From 84e39d0cfa83c07676654993b81fbe0f4af400e9 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 2 Oct 2024 14:21:36 +0100 Subject: [PATCH 01/80] EES-5552 - adding test as an additional branch from which to perform deploys --- azure-pipelines.dfe.yml | 3 +++ infrastructure/infrastructure-pipeline.yml | 1 + 2 files changed, 4 insertions(+) diff --git a/azure-pipelines.dfe.yml b/azure-pipelines.dfe.yml index fe47c44195b..c3222bf738c 100644 --- a/azure-pipelines.dfe.yml +++ b/azure-pipelines.dfe.yml @@ -5,6 +5,7 @@ parameters: default: - master - dev + - test variables: BuildConfiguration: 'Release' @@ -18,12 +19,14 @@ trigger: include: - master - dev + - test paths: exclude: - infrastructure/ pr: - master - dev + - test jobs: - job: 'Backend' diff --git a/infrastructure/infrastructure-pipeline.yml b/infrastructure/infrastructure-pipeline.yml index d62d0511828..4a291d4ceb6 100644 --- a/infrastructure/infrastructure-pipeline.yml +++ b/infrastructure/infrastructure-pipeline.yml @@ -4,6 +4,7 @@ trigger: include: - master - dev + - test paths: include: - infrastructure/* From 321b384973aebc531a0f730daaaed32675e1395d Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Mon, 23 Sep 2024 01:54:08 +0100 Subject: [PATCH 02/80] EES-5494 Fix example for `DataSetVersionViewModel.GeographicLevels` --- .../ViewModels/DataSetVersionViewModels.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetVersionViewModels.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetVersionViewModels.cs index c275b94b3c3..da7296dc37b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetVersionViewModels.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetVersionViewModels.cs @@ -82,7 +82,7 @@ public class DataSetVersionViewModel /// /// The geographic levels available in the data set. /// - /// ["NAT", "REG", "LA"] + /// ["National", "Regional", "Local authority"] [JsonConverter(typeof(ReadOnlyListJsonConverter>))] public required IReadOnlyList GeographicLevels { get; init; } From 99dd0bcf8e690ad73997b3cb31cf4fed74eeb739 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Mon, 23 Sep 2024 02:01:20 +0100 Subject: [PATCH 03/80] EES-5494 Add missing examples for `DataSetQueryResultViewModel` properties --- .../DataSetQueryResultViewModels.cs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetQueryResultViewModels.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetQueryResultViewModels.cs index c305f9b110d..d5f390f5903 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetQueryResultViewModels.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetQueryResultViewModels.cs @@ -37,6 +37,13 @@ public record DataSetQueryResultViewModel /// This is a dictionary where the key is the location's geographic /// level and the value is the location's ID. /// + /// + /// { + /// "NAT": "04bTr", + /// "REG": "4veOu", + /// "LA": "owqlK" + /// } + /// public required Dictionary Locations { get; init; } /// @@ -45,6 +52,13 @@ public record DataSetQueryResultViewModel /// This is a dictionary where the key is the filter ID and /// the value is the specific filter option ID. /// + /// + /// { + /// "ups2K": "n0WqP", + /// "j51wV": "AnZsi", + /// "hAkBQ": "dvB4z" + /// } + /// public required Dictionary Filters { get; init; } /// @@ -53,5 +67,12 @@ public record DataSetQueryResultViewModel /// This is a dictionary where the key is the indicator ID /// and the value is the data value. /// + /// + /// { + /// "wLcft": "23593018", + /// "4S8Ou": "50.342", + /// "9kVFg": "25369172" + /// } + /// public required Dictionary Values { get; init; } } From 89e31696907832eabab53e609e1b0a230e912958 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Mon, 23 Sep 2024 02:04:00 +0100 Subject: [PATCH 04/80] EES-5494 Shorten name for `GeographicLevelChangeViewModel` --- .../Services/DataSetVersionChangeService.cs | 16 ++++++++-------- .../ViewModels/ChangeViewModels.cs | 2 +- .../ViewModels/DataSetVersionChangesViewModel.cs | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Services/DataSetVersionChangeService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Services/DataSetVersionChangeService.cs index 267f463fe18..86358ae7670 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Services/DataSetVersionChangeService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Services/DataSetVersionChangeService.cs @@ -64,8 +64,8 @@ private static DataSetVersionChangesViewModel MapChanges(DataSetVersion dataSetV GetFilterChanges(dataSetVersion.FilterMetaChanges); var filterOptionChanges = GetFilterOptionChanges(dataSetVersion.FilterOptionMetaChanges); - var geographicLevelOptionChange = - GetGeographicLevelOptionChange(dataSetVersion.GeographicLevelMetaChange); + var geographicLevelChanges = + GetGeographicLevelChanges(dataSetVersion.GeographicLevelMetaChange); var indicatorChanges = GetIndicatorChanges(dataSetVersion.IndicatorMetaChanges); var locationGroupChanges = @@ -81,7 +81,7 @@ private static DataSetVersionChangesViewModel MapChanges(DataSetVersion dataSetV { Filters = filterChanges?.GetValueOrDefault(ChangeType.Major), FilterOptions = filterOptionChanges?.GetValueOrDefault(ChangeType.Major), - GeographicLevels = geographicLevelOptionChange?.GetValueOrDefault(ChangeType.Major), + GeographicLevels = geographicLevelChanges?.GetValueOrDefault(ChangeType.Major), Indicators = indicatorChanges?.GetValueOrDefault(ChangeType.Major), LocationGroups = locationGroupChanges?.GetValueOrDefault(ChangeType.Major), LocationOptions = locationOptionChanges?.GetValueOrDefault(ChangeType.Major), @@ -91,7 +91,7 @@ private static DataSetVersionChangesViewModel MapChanges(DataSetVersion dataSetV { Filters = filterChanges?.GetValueOrDefault(ChangeType.Minor), FilterOptions = filterOptionChanges?.GetValueOrDefault(ChangeType.Minor), - GeographicLevels = geographicLevelOptionChange?.GetValueOrDefault(ChangeType.Minor), + GeographicLevels = geographicLevelChanges?.GetValueOrDefault(ChangeType.Minor), Indicators = indicatorChanges?.GetValueOrDefault(ChangeType.Minor), LocationGroups = locationGroupChanges?.GetValueOrDefault(ChangeType.Minor), LocationOptions = locationOptionChanges?.GetValueOrDefault(ChangeType.Minor), @@ -174,7 +174,7 @@ private static DataSetVersionChangesViewModel MapChanges(DataSetVersion dataSetV : null; } - private static Dictionary>? GetGeographicLevelOptionChange( + private static Dictionary>? GetGeographicLevelChanges( GeographicLevelMetaChange? change) { if (change is null) @@ -190,17 +190,17 @@ private static DataSetVersionChangesViewModel MapChanges(DataSetVersion dataSetV return null; } - List changes = + List changes = [ ..currentLevels .Where(level => !previousLevels.Contains(level)) - .Select(level => new GeographicLevelOptionChangeViewModel + .Select(level => new GeographicLevelChangeViewModel { CurrentState = GeographicLevelViewModel.Create(level), }), ..previousLevels .Where(level => !currentLevels.Contains(level)) - .Select(level => new GeographicLevelOptionChangeViewModel + .Select(level => new GeographicLevelChangeViewModel { PreviousState = GeographicLevelViewModel.Create(level), }), diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/ChangeViewModels.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/ChangeViewModels.cs index 2da52126dc4..f5a8148fcf0 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/ChangeViewModels.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/ChangeViewModels.cs @@ -55,7 +55,7 @@ public override bool IsMajor() /// /// A change to a geographic level option in a data set. /// -public record GeographicLevelOptionChangeViewModel : ChangeViewModel +public record GeographicLevelChangeViewModel : ChangeViewModel { public override bool IsMajor() => CurrentState is null && PreviousState is not null; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetVersionChangesViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetVersionChangesViewModel.cs index 987fe343966..eac2dc62be5 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetVersionChangesViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetVersionChangesViewModel.cs @@ -34,7 +34,7 @@ public record ChangeSetViewModel /// /// A list of any geographic level changes made to the data set. /// - public IReadOnlyList? GeographicLevels { get; init; } + public IReadOnlyList? GeographicLevels { get; init; } /// /// A list of any indicator changes made to the data set. From 53ab42c227ce7ba40a477ef96f7ecbf16d53ee2d Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Mon, 23 Sep 2024 12:39:25 +0100 Subject: [PATCH 05/80] EES-5494 Update examples for various public API view models --- .../Requests/DataSetQueryCriteriaAnd.cs | 5 ++++- .../Requests/DataSetQueryCriteriaOr.cs | 5 ++++- .../ViewModels/DataSetVersionViewModels.cs | 4 ++-- .../ViewModels/DataSetViewModels.cs | 10 +++++----- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Requests/DataSetQueryCriteriaAnd.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Requests/DataSetQueryCriteriaAnd.cs index fae2ffd81b1..972d8a33f3f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Requests/DataSetQueryCriteriaAnd.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Requests/DataSetQueryCriteriaAnd.cs @@ -22,7 +22,10 @@ public record DataSetQueryCriteriaAnd : IDataSetQueryCriteria /// }, /// { /// "locations": { - /// "eq": "LA|code|E08000019" + /// "eq": { + /// "level": "LA", + /// "code": "E08000019" + /// } /// } /// } /// ] diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Requests/DataSetQueryCriteriaOr.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Requests/DataSetQueryCriteriaOr.cs index dfb53433884..8ba0d5b1e43 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Requests/DataSetQueryCriteriaOr.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Requests/DataSetQueryCriteriaOr.cs @@ -22,7 +22,10 @@ public record DataSetQueryCriteriaOr : IDataSetQueryCriteria /// }, /// { /// "locations": { - /// "eq": "LA|code|E08000019" + /// "eq": { + /// "level": "LA", + /// "code": "E08000019" + /// } /// } /// } /// ] diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetVersionViewModels.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetVersionViewModels.cs index da7296dc37b..a9c4b87a4bf 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetVersionViewModels.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetVersionViewModels.cs @@ -49,7 +49,7 @@ public class DataSetVersionViewModel /// /// When the version was withdrawn. /// - /// null + /// 2024-06-01T12:00:00+00:00 public DateTimeOffset? Withdrawn { get; init; } /// @@ -95,7 +95,7 @@ public class DataSetVersionViewModel /// /// The indicators available in the data set. /// - /// ["Authorised absence rate" "Overall absence rate"] + /// ["Authorised absence rate", "Overall absence rate"] public required IReadOnlyList Indicators { get; init; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetViewModels.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetViewModels.cs index 00910076c05..cce5ba65818 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetViewModels.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/DataSetViewModels.cs @@ -36,15 +36,15 @@ public record DataSetViewModel public required DataSetStatus Status { get; init; } /// - /// The latest published data set version. + /// The ID of the data set that supersedes this data set (if it has been deprecated). /// - public required DataSetLatestVersionViewModel LatestVersion { get; init; } + /// 2118a6df-4934-4a1f-ad2e-4589d2b9ccaf + public Guid? SupersedingDataSetId { get; init; } /// - /// The ID of the data set that supersedes this data set (if it has been deprecated). + /// The latest published data set version. /// - /// null - public Guid? SupersedingDataSetId { get; init; } + public required DataSetLatestVersionViewModel LatestVersion { get; init; } } /// From 703ce51bce5b5ff8aba42ee8aec662d19f06c906 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Mon, 23 Sep 2024 17:08:01 +0100 Subject: [PATCH 06/80] EES-5494 Add `EnumToEnumLabelJsonConverter` (Newtonsoft.Json) --- .../EnumToEnumLabelJsonConverterTests.cs | 46 +++++++++++++++++++ .../EnumToEnumLabelJsonConverter.cs | 28 +++++++++++ 2 files changed, 74 insertions(+) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Converters/EnumToEnumLabelJsonConverterTests.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Common/Converters/EnumToEnumLabelJsonConverter.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Converters/EnumToEnumLabelJsonConverterTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Converters/EnumToEnumLabelJsonConverterTests.cs new file mode 100644 index 00000000000..494c3f2e2a0 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Converters/EnumToEnumLabelJsonConverterTests.cs @@ -0,0 +1,46 @@ +#nullable enable +using GovUk.Education.ExploreEducationStatistics.Common.Converters; +using GovUk.Education.ExploreEducationStatistics.Common.Database; +using Newtonsoft.Json; +using Xunit; + +namespace GovUk.Education.ExploreEducationStatistics.Common.Tests.Converters; + +public class EnumToEnumLabelJsonConverterTests +{ + private enum SampleEnum + { + [EnumLabelValue("SampleLabel", "SampleValue")] + Sample + } + + private record SampleClass + { + [JsonConverter(typeof(EnumToEnumLabelJsonConverter))] + public SampleEnum SampleField { get; init; } + } + + [Fact] + public void SerializeObject() + { + var objectToSerialize = new SampleClass + { + SampleField = SampleEnum.Sample + }; + + Assert.Equal("{\"SampleField\":\"SampleLabel\"}", JsonConvert.SerializeObject(objectToSerialize)); + } + + [Fact] + public void DeserializeObject() + { + const string jsonText = "{\"SampleField\":\"SampleLabel\"}"; + + var expected = new SampleClass + { + SampleField = SampleEnum.Sample + }; + + Assert.Equal(expected, JsonConvert.DeserializeObject(jsonText)); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Converters/EnumToEnumLabelJsonConverter.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Converters/EnumToEnumLabelJsonConverter.cs new file mode 100644 index 00000000000..2bf23207453 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Converters/EnumToEnumLabelJsonConverter.cs @@ -0,0 +1,28 @@ +#nullable enable +using System; +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; +using Newtonsoft.Json; + +namespace GovUk.Education.ExploreEducationStatistics.Common.Converters; + +public class EnumToEnumLabelJsonConverter : JsonConverter where TEnum : Enum +{ + public override void WriteJson(JsonWriter writer, TEnum? value, JsonSerializer serializer) + { + if (value != null) + { + writer.WriteValue(value.GetEnumLabel()); + } + } + + public override TEnum ReadJson( + JsonReader reader, + Type objectType, + TEnum? existingValue, + bool hasExistingValue, + JsonSerializer serializer) + { + return EnumUtil.GetFromEnumLabel(reader.Value?.ToString() ?? string.Empty); + } +} From 81d10457894fb4617b62d65ac4438756ba5a4949 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Mon, 23 Sep 2024 17:08:16 +0100 Subject: [PATCH 07/80] EES-5494 Cleanup `EnumToEnumValueJsonConverter` and its tests --- .../EnumToEnumValueJsonConverterTests.cs | 80 +++++++------------ .../EnumToEnumValueJsonConverter.cs | 28 ++++--- 2 files changed, 47 insertions(+), 61 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Converters/EnumToEnumValueJsonConverterTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Converters/EnumToEnumValueJsonConverterTests.cs index de5d48a8a49..1f755e7e388 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Converters/EnumToEnumValueJsonConverterTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Converters/EnumToEnumValueJsonConverterTests.cs @@ -1,64 +1,46 @@ +#nullable enable using GovUk.Education.ExploreEducationStatistics.Common.Converters; using GovUk.Education.ExploreEducationStatistics.Common.Database; using Newtonsoft.Json; using Xunit; -namespace GovUk.Education.ExploreEducationStatistics.Common.Tests.Converters +namespace GovUk.Education.ExploreEducationStatistics.Common.Tests.Converters; + +public class EnumToEnumValueJsonConverterTests { - public class EnumToEnumValueJsonConverterTests + private enum SampleEnum { - private enum SampleEnum - { - [EnumLabelValue("SampleLabel", "SampleValue")] - Sample - } - - private class SampleClass - { - [JsonConverter(typeof(EnumToEnumValueJsonConverter))] - public SampleEnum SampleField { get; set; } - - protected bool Equals(SampleClass other) - { - return SampleField == other.SampleField; - } - - public override bool Equals(object obj) - { - if (ReferenceEquals(null, obj)) return false; - if (ReferenceEquals(this, obj)) return true; - if (obj.GetType() != this.GetType()) return false; - return Equals((SampleClass) obj); - } + [EnumLabelValue("SampleLabel", "SampleValue")] + Sample + } - public override int GetHashCode() - { - return (int) SampleField; - } - } + private record SampleClass + { + [JsonConverter(typeof(EnumToEnumValueJsonConverter))] + public SampleEnum SampleField { get; init; } + } - [Fact] - public void SerializeObject() + [Fact] + public void SerializeObject() + { + var objectToSerialize = new SampleClass { - var objectToSerialize = new SampleClass - { - SampleField = SampleEnum.Sample - }; + SampleField = SampleEnum.Sample + }; - Assert.Equal("{\"SampleField\":\"SampleValue\"}", JsonConvert.SerializeObject(objectToSerialize)); - } + Assert.Equal("{\"SampleField\":\"SampleValue\"}", JsonConvert.SerializeObject(objectToSerialize)); + } - [Fact] - public void DeserializeObject() - { - const string jsonText = "{\"SampleField\":\"SampleValue\"}"; + [Fact] + public void DeserializeObject() + { + const string jsonText = "{\"SampleField\":\"SampleValue\"}"; - var expected = new SampleClass - { - SampleField = SampleEnum.Sample - }; + var expected = new SampleClass + { + SampleField = SampleEnum.Sample + }; - Assert.Equal(expected, JsonConvert.DeserializeObject(jsonText)); - } + Assert.Equal(expected, JsonConvert.DeserializeObject(jsonText)); } -} \ No newline at end of file +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Converters/EnumToEnumValueJsonConverter.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Converters/EnumToEnumValueJsonConverter.cs index 90b9d46d3f3..93ae0784ce6 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Converters/EnumToEnumValueJsonConverter.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Converters/EnumToEnumValueJsonConverter.cs @@ -1,24 +1,28 @@ +#nullable enable using System; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Utils; using Newtonsoft.Json; -namespace GovUk.Education.ExploreEducationStatistics.Common.Converters +namespace GovUk.Education.ExploreEducationStatistics.Common.Converters; + +public class EnumToEnumValueJsonConverter : JsonConverter where TEnum : Enum { - public class EnumToEnumValueJsonConverter : JsonConverter where TEnum : Enum + public override void WriteJson(JsonWriter writer, TEnum? value, JsonSerializer serializer) { - public override void WriteJson(JsonWriter writer, TEnum value, JsonSerializer serializer) + if (value != null) { - if (value != null) - { - writer.WriteValue(value.GetEnumValue()); - } + writer.WriteValue(value.GetEnumValue()); } + } - public override TEnum ReadJson(JsonReader reader, Type objectType, TEnum existingValue, bool hasExistingValue, - JsonSerializer serializer) - { - return EnumUtil.GetFromEnumValue(reader.Value.ToString()); - } + public override TEnum ReadJson( + JsonReader reader, + Type objectType, + TEnum? existingValue, + bool hasExistingValue, + JsonSerializer serializer) + { + return EnumUtil.GetFromEnumValue(reader.Value?.ToString() ?? string.Empty); } } From e3fffa1d147fbc79a4ec5e4407b0495e16a08fbe Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Tue, 24 Sep 2024 00:04:11 +0100 Subject: [PATCH 08/80] EES-5494 Rename IndicatorUnit `Numberstring` and `None` enums This changes the IndicatorUnit `numberstring` enum to be displayed as `string` to end users (e.g. for the public API), but still be stored internally as `numberstring`. This is likely to have been decided to avoid having to tell analysts to change again to use `string`. Additionally, the `IndicatorUnit.Number` enum has been changed to `None` as this moreclosely reflects its actual semantics as we also allow non-numeric values as well. --- .../Model/IndicatorUnit.cs | 39 +++++++++++-------- .../Fixtures/IndicatorGeneratorExtensions.cs | 2 +- .../Indicator.cs | 2 +- .../SubjectMetaServiceTests.cs | 4 +- .../Meta/IndicatorCsvMetaViewModel.cs | 2 +- .../Meta/IndicatorMetaViewModels.cs | 2 +- .../number/__tests__/formatPretty.test.ts | 4 +- .../src/utils/number/formatPretty.ts | 2 +- 8 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Model/IndicatorUnit.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Model/IndicatorUnit.cs index 3084b0120de..4356cd20b89 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Model/IndicatorUnit.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Model/IndicatorUnit.cs @@ -1,25 +1,32 @@ +#nullable enable using GovUk.Education.ExploreEducationStatistics.Common.Database; -namespace GovUk.Education.ExploreEducationStatistics.Common.Model +namespace GovUk.Education.ExploreEducationStatistics.Common.Model; + +public enum IndicatorUnit { - public enum IndicatorUnit - { - [EnumLabelValue("", "")] - Number, + [EnumLabelValue("", "")] + None, - [EnumLabelValue("%", "%")] - Percent, + [EnumLabelValue("%", "%")] + Percent, - [EnumLabelValue("£", "£")] - Pound, + [EnumLabelValue("£", "£")] + Pound, - [EnumLabelValue("£m", "£m")] - MillionPounds, + [EnumLabelValue("£m", "£m")] + MillionPounds, - [EnumLabelValue("pp", "pp")] - PercentagePoint, + [EnumLabelValue("pp", "pp")] + PercentagePoint, - [EnumLabelValue("numberstring", "numberstring")] - NumberString, - } + // We named this `numberstring` in EES-5478, but this is + // likely to be confusing for end users who can see this unit + // (e.g. in the public API). + // It's really just saying 'this is a string without any numeric + // formatting' so `string` is a more appropriate name. + // To compromise, we'll display `string` to end users and persist + // as `numberstring` internally (which analysts will publish with). + [EnumLabelValue("string", "numberstring")] + String, } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Model.Tests/Fixtures/IndicatorGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Model.Tests/Fixtures/IndicatorGeneratorExtensions.cs index 557534e5eba..9405bacba41 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Model.Tests/Fixtures/IndicatorGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Model.Tests/Fixtures/IndicatorGeneratorExtensions.cs @@ -34,7 +34,7 @@ public static InstanceSetters SetDefaults(this InstanceSetters i.Label) .SetDefault(i => i.Name) .Set(i => i.Name, (_, i) => i.Name.SnakeCase()) - .Set(i => i.Unit, IndicatorUnit.Number) + .Set(i => i.Unit, IndicatorUnit.None) .Set(i => i.DecimalPlaces, 0); public static InstanceSetters SetLabel( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Model/Indicator.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Model/Indicator.cs index 4ce2f805fff..1e7010ef711 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Model/Indicator.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Model/Indicator.cs @@ -10,7 +10,7 @@ public class Indicator public Guid Id { get; set; } public string Label { get; set; } = string.Empty; public string Name { get; set; } = string.Empty; - public IndicatorUnit Unit { get; set; } = IndicatorUnit.Number; + public IndicatorUnit Unit { get; set; } = IndicatorUnit.None; public IndicatorGroup IndicatorGroup { get; set; } = null!; public Guid IndicatorGroupId { get; set; } public List Footnotes { get; set; } = new(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/SubjectMetaServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/SubjectMetaServiceTests.cs index 9dc5be4011c..90d7b753b8a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/SubjectMetaServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.Services.Tests/SubjectMetaServiceTests.cs @@ -757,7 +757,7 @@ await statisticsDbContext.MatchedObservations.AddRangeAsync( new Indicator { Id = Guid.NewGuid(), - Unit = IndicatorUnit.Number, + Unit = IndicatorUnit.None, Label = "Indicator 2" }) }, @@ -887,7 +887,7 @@ await statisticsDbContext.MatchedObservations.AddRangeAsync( new IndicatorMetaViewModel { Label = "Indicator 2", - Unit = IndicatorUnit.Number, + Unit = IndicatorUnit.None, Value = indicatorGroups[0].Indicators[0].Id }), Order = 1 diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.ViewModels/Meta/IndicatorCsvMetaViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.ViewModels/Meta/IndicatorCsvMetaViewModel.cs index 76bb7624577..89987e19464 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.ViewModels/Meta/IndicatorCsvMetaViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.ViewModels/Meta/IndicatorCsvMetaViewModel.cs @@ -11,7 +11,7 @@ public record IndicatorCsvMetaViewModel public string Label { get; init; } = string.Empty; - [JsonConverter(typeof(EnumToEnumValueJsonConverter))] + [JsonConverter(typeof(EnumToEnumLabelJsonConverter))] public IndicatorUnit Unit { get; init; } public string Name { get; init; } = string.Empty; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Data.ViewModels/Meta/IndicatorMetaViewModels.cs b/src/GovUk.Education.ExploreEducationStatistics.Data.ViewModels/Meta/IndicatorMetaViewModels.cs index 74639574a0f..587f5368d9c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Data.ViewModels/Meta/IndicatorMetaViewModels.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Data.ViewModels/Meta/IndicatorMetaViewModels.cs @@ -28,7 +28,7 @@ public record IndicatorMetaViewModel { public string Label { get; init; } = string.Empty; - [JsonConverter(typeof(EnumToEnumValueJsonConverter))] + [JsonConverter(typeof(EnumToEnumLabelJsonConverter))] public IndicatorUnit Unit { get; init; } public Guid Value { get; init; } diff --git a/src/explore-education-statistics-common/src/utils/number/__tests__/formatPretty.test.ts b/src/explore-education-statistics-common/src/utils/number/__tests__/formatPretty.test.ts index c27677bb989..00762375d2e 100644 --- a/src/explore-education-statistics-common/src/utils/number/__tests__/formatPretty.test.ts +++ b/src/explore-education-statistics-common/src/utils/number/__tests__/formatPretty.test.ts @@ -79,8 +79,8 @@ describe('formatPretty', () => { expect(formatPretty(15.1234, '%', 2)).toBe('15.12%'); expect(formatPretty(150000000.1234, '', 4)).toBe('150,000,000.1234'); - expect(formatPretty(150000000.1234, 'numberstring')).toBe('150000000.1234'); - expect(formatPretty('ABCD123456789', 'numberstring')).toBe('ABCD123456789'); + expect(formatPretty(150000000.1234, 'string')).toBe('150000000.1234'); + expect(formatPretty('ABCD123456789', 'string')).toBe('ABCD123456789'); }); test('returns NaN string if number value is not a number', () => { diff --git a/src/explore-education-statistics-common/src/utils/number/formatPretty.ts b/src/explore-education-statistics-common/src/utils/number/formatPretty.ts index 90dc8b3da46..fba00aa883a 100644 --- a/src/explore-education-statistics-common/src/utils/number/formatPretty.ts +++ b/src/explore-education-statistics-common/src/utils/number/formatPretty.ts @@ -20,7 +20,7 @@ export default function formatPretty( decimalPlaces?: number, ): string { switch (unit) { - case 'numberstring': + case 'string': return String(value); case '£': { const formattedNumber = formatNumber(value, decimalPlaces); From 80ad958f04c6e3b174f17ffe857ce964e5be3f0c Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Tue, 24 Sep 2024 00:04:24 +0100 Subject: [PATCH 09/80] EES-5494 Add OpenAPI docs for `IndicatorUnit` --- .../Swagger/IndicatorUnitSchemaFilterTests.cs | 49 ++++++++++++ .../Swagger/JsonConverterSchemaFilterTests.cs | 77 ++++++++++++++----- .../Swagger/GeographicLevelSchemaFilter.cs | 3 +- .../Swagger/IndicatorUnitSchemaFilter.cs | 37 +++++++++ .../Swagger/JsonConverterSchemaFilter.cs | 1 + .../Swagger/SwaggerConfig.cs | 1 + .../Swagger/TimeIdentifierSchemaFilter.cs | 2 +- .../ViewModels/IndicatorViewModel.cs | 4 +- .../Functions/ImportMetadataFunctionTests.cs | 2 +- 9 files changed, 150 insertions(+), 26 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Swagger/IndicatorUnitSchemaFilterTests.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/IndicatorUnitSchemaFilter.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Swagger/IndicatorUnitSchemaFilterTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Swagger/IndicatorUnitSchemaFilterTests.cs new file mode 100644 index 00000000000..2c63952a6cc --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Swagger/IndicatorUnitSchemaFilterTests.cs @@ -0,0 +1,49 @@ +using System.Text.Json; +using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Swagger; +using Microsoft.OpenApi.Any; +using Microsoft.OpenApi.Models; +using Swashbuckle.AspNetCore.SwaggerGen; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Swagger; + +public class IndicatorUnitSchemaFilterTests +{ + private readonly SchemaGenerator _schemaGenerator = new( + new SchemaGeneratorOptions + { + UseAllOfToExtendReferenceSchemas = true, + SchemaFilters = [new IndicatorUnitSchemaFilter()], + }, + new JsonSerializerDataContractResolver(new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }) + ); + + private readonly SchemaRepository _schemaRepository = new("Default"); + + [Fact] + public void CorrectSchema() + { + var schema = GenerateSchema(); + var indicatorUnits = EnumUtil.GetEnumLabels().ToHashSet(); + + Assert.Equal("string", schema.Type); + Assert.Equal(indicatorUnits.Count, schema.Enum.Count); + + Assert.All(schema.Enum, e => + { + var enumString = Assert.IsType(e); + Assert.Contains(enumString.Value, indicatorUnits); + }); + } + + private OpenApiSchema GenerateSchema() + { + _schemaGenerator.GenerateSchema(typeof(IndicatorUnit), _schemaRepository); + + return _schemaRepository.Schemas[nameof(IndicatorUnit)]; + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Swagger/JsonConverterSchemaFilterTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Swagger/JsonConverterSchemaFilterTests.cs index 6564bbd6cc4..5cd91f27e66 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Swagger/JsonConverterSchemaFilterTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Swagger/JsonConverterSchemaFilterTests.cs @@ -15,7 +15,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests.Swagg public class JsonConverterSchemaFilterTests { - private readonly ISchemaGenerator _schemaGenerator = new SchemaGenerator( + private readonly SchemaGenerator _schemaGenerator = new( new SchemaGeneratorOptions { UseAllOfToExtendReferenceSchemas = true, @@ -141,13 +141,13 @@ public void ListConverter_GeographicLevelEnumValues() } [Theory] - [InlineData(nameof(TestIgnoredEnumConverters.GeographicLevelStringEnum))] - [InlineData(nameof(TestIgnoredEnumConverters.GeographicLevelStringEnumTyped))] - [InlineData(nameof(TestIgnoredEnumConverters.GeographicLevelEnumToEnumLabel))] - [InlineData(nameof(TestIgnoredEnumConverters.GeographicLevelEnumToEnumValue))] + [InlineData(nameof(TestIgnoredGeographicLevelConverters.StringEnum))] + [InlineData(nameof(TestIgnoredGeographicLevelConverters.StringEnumTyped))] + [InlineData(nameof(TestIgnoredGeographicLevelConverters.EnumToEnumLabel))] + [InlineData(nameof(TestIgnoredGeographicLevelConverters.EnumToEnumValue))] public void GeographicLevel_Ignored(string propertyName) { - var schema = GenerateSchema(); + var schema = GenerateSchema(); var schemaPropertyName = propertyName.CamelCase(); var propertySchema = schema.Properties[schemaPropertyName]; @@ -159,13 +159,31 @@ public void GeographicLevel_Ignored(string propertyName) } [Theory] - [InlineData(nameof(TestIgnoredEnumConverters.TimeIdentifierStringEnum))] - [InlineData(nameof(TestIgnoredEnumConverters.TimeIdentifierStringEnumTyped))] - [InlineData(nameof(TestIgnoredEnumConverters.TimeIdentifierEnumToEnumLabel))] - [InlineData(nameof(TestIgnoredEnumConverters.TimeIdentifierEnumToEnumValue))] + [InlineData(nameof(TestIgnoredIndicatorUnitConverters.StringEnum))] + [InlineData(nameof(TestIgnoredIndicatorUnitConverters.StringEnumTyped))] + [InlineData(nameof(TestIgnoredIndicatorUnitConverters.EnumToEnumLabel))] + [InlineData(nameof(TestIgnoredIndicatorUnitConverters.EnumToEnumValue))] + public void IndicatorUnit_Ignored(string propertyName) + { + var schema = GenerateSchema(); + + var schemaPropertyName = propertyName.CamelCase(); + var propertySchema = schema.Properties[schemaPropertyName]; + + Assert.NotEmpty(propertySchema.AllOf); + + Assert.Null(propertySchema.Type); + Assert.Empty(propertySchema.Enum); + } + + [Theory] + [InlineData(nameof(TestIgnoredTimeIdentifierConverters.StringEnum))] + [InlineData(nameof(TestIgnoredTimeIdentifierConverters.StringEnumTyped))] + [InlineData(nameof(TestIgnoredTimeIdentifierConverters.EnumToEnumLabel))] + [InlineData(nameof(TestIgnoredTimeIdentifierConverters.EnumToEnumValue))] public void TimeIdentifier_Ignored(string propertyName) { - var schema = GenerateSchema(); + var schema = GenerateSchema(); var schemaPropertyName = propertyName.CamelCase(); var propertySchema = schema.Properties[schemaPropertyName]; @@ -176,6 +194,7 @@ public void TimeIdentifier_Ignored(string propertyName) Assert.Empty(propertySchema.Enum); } + private OpenApiSchema GenerateSchema() { _schemaGenerator.GenerateSchema(typeof(TestEnum), _schemaRepository); @@ -199,31 +218,49 @@ private class TestEnumConverters public TestEnum EnumToEnumValue { get; init; } } - private class TestIgnoredEnumConverters + private class TestIgnoredGeographicLevelConverters { [JsonConverter(typeof(JsonStringEnumConverter))] - public GeographicLevel GeographicLevelStringEnum { get; init; } + public GeographicLevel StringEnum { get; init; } [JsonConverter(typeof(JsonStringEnumConverter))] - public GeographicLevel GeographicLevelStringEnumTyped { get; init; } + public GeographicLevel StringEnumTyped { get; init; } [JsonConverter(typeof(EnumToEnumLabelJsonConverter))] - public GeographicLevel GeographicLevelEnumToEnumLabel { get; init; } + public GeographicLevel EnumToEnumLabel { get; init; } [JsonConverter(typeof(EnumToEnumValueJsonConverter))] - public GeographicLevel GeographicLevelEnumToEnumValue { get; init; } + public GeographicLevel EnumToEnumValue { get; init; } + } + + private class TestIgnoredIndicatorUnitConverters + { + [JsonConverter(typeof(JsonStringEnumConverter))] + public IndicatorUnit StringEnum { get; init; } + [JsonConverter(typeof(JsonStringEnumConverter))] + public IndicatorUnit StringEnumTyped { get; init; } + + [JsonConverter(typeof(EnumToEnumLabelJsonConverter))] + public IndicatorUnit EnumToEnumLabel { get; init; } + + [JsonConverter(typeof(EnumToEnumValueJsonConverter))] + public IndicatorUnit EnumToEnumValue { get; init; } + } + + private class TestIgnoredTimeIdentifierConverters + { [JsonConverter(typeof(JsonStringEnumConverter))] - public TimeIdentifier TimeIdentifierStringEnum { get; init; } + public TimeIdentifier StringEnum { get; init; } [JsonConverter(typeof(JsonStringEnumConverter))] - public TimeIdentifier TimeIdentifierStringEnumTyped { get; init; } + public TimeIdentifier StringEnumTyped { get; init; } [JsonConverter(typeof(EnumToEnumLabelJsonConverter))] - public TimeIdentifier TimeIdentifierEnumToEnumLabel { get; init; } + public TimeIdentifier EnumToEnumLabel { get; init; } [JsonConverter(typeof(EnumToEnumValueJsonConverter))] - public TimeIdentifier TimeIdentifierEnumToEnumValue { get; init; } + public TimeIdentifier EnumToEnumValue { get; init; } } private class TestReadOnlyListEnumConverters diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/GeographicLevelSchemaFilter.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/GeographicLevelSchemaFilter.cs index 83f188d68d1..d3b30d0671b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/GeographicLevelSchemaFilter.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/GeographicLevelSchemaFilter.cs @@ -1,4 +1,3 @@ -using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; using GovUk.Education.ExploreEducationStatistics.Common.Utils; using Microsoft.OpenApi.Any; @@ -40,7 +39,7 @@ public void Apply(OpenApiSchema schema, SchemaFilterContext context) - `SCH` - School - `SPON` - Sponsor - `WARD` - Ward - """.TrimIndent(); + """; schema.Example = new OpenApiString("NAT"); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/IndicatorUnitSchemaFilter.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/IndicatorUnitSchemaFilter.cs new file mode 100644 index 00000000000..06fc3633c7c --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/IndicatorUnitSchemaFilter.cs @@ -0,0 +1,37 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.Utils; +using Microsoft.OpenApi.Any; +using Microsoft.OpenApi.Models; +using Swashbuckle.AspNetCore.SwaggerGen; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Swagger; + +public class IndicatorUnitSchemaFilter : ISchemaFilter +{ + public void Apply(OpenApiSchema schema, SchemaFilterContext context) + { + if (context.MemberInfo == null && context.Type == typeof(IndicatorUnit)) + { + schema.Type = "string"; + schema.Description = + """ + The recommended unit to format an indicator with. + + The allowed values are: + + - `""` - No units with numeric formatting where possible e.g. `123,500,600` + - `£` - Pound sterling with numeric formatting e.g. `£1,234,500` + - `£m` - Pound sterling in millions with numeric formatting e.g. `£1,234m` + - `%` - Percentage with numeric formatting e.g. `1,234.56%` + - `pp` - Percentage points with numeric formatting e.g. `1,234pp` + - `string` - String with no units and no numeric formatting e.g. `123500600` + """; + + schema.Example = new OpenApiString("%"); + + schema.Enum = EnumUtil.GetEnumLabels() + .Select(unit => new OpenApiString(unit)) + .ToList(); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/JsonConverterSchemaFilter.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/JsonConverterSchemaFilter.cs index bd3b235ba8f..6f39b5cc628 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/JsonConverterSchemaFilter.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/JsonConverterSchemaFilter.cs @@ -22,6 +22,7 @@ internal class JsonConverterSchemaFilter : ISchemaFilter private readonly HashSet _typesToIgnore = [ typeof(GeographicLevel), + typeof(IndicatorUnit), typeof(TimeIdentifier), ]; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/SwaggerConfig.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/SwaggerConfig.cs index 21df413010c..8d00439bc1e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/SwaggerConfig.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/SwaggerConfig.cs @@ -25,6 +25,7 @@ public void Configure(SwaggerGenOptions options) options.SchemaFilter(); options.SchemaFilter(); options.SchemaFilter(); + options.SchemaFilter(); options.SchemaFilter(); Directory diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/TimeIdentifierSchemaFilter.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/TimeIdentifierSchemaFilter.cs index 250a4770ed3..4a429292580 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/TimeIdentifierSchemaFilter.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Swagger/TimeIdentifierSchemaFilter.cs @@ -39,7 +39,7 @@ The code identifying the time period's type. - `TYQ1 - FYQ4` - tax year quarter 1 to 4 - `W1 - W52` - week 1 to 52 - `M1 - M12` - month 1 to 12 - """.TrimIndent(); + """; schema.Example = new OpenApiString("CY"); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/IndicatorViewModel.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/IndicatorViewModel.cs index 69b0611571a..9c502538046 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/IndicatorViewModel.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/ViewModels/IndicatorViewModel.cs @@ -29,14 +29,14 @@ public record IndicatorViewModel public required string Label { get; init; } /// - /// A numeric unit for an indicator. + /// The type of unit that should be used when formatting the indicator. /// /// % [JsonConverter(typeof(EnumToEnumLabelJsonConverter))] public required IndicatorUnit? Unit { get; init; } /// - /// The optimal number of decimal places that the indicator should use when displayed. + /// The recommended number of decimal places to use when formatting the indicator. /// /// 2 public int? DecimalPlaces { get; init; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportMetadataFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportMetadataFunctionTests.cs index 0c6943cdca9..d3b7b45492d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportMetadataFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportMetadataFunctionTests.cs @@ -689,7 +689,7 @@ public async Task DuckDbMeta_CorrectIndicators(ProcessorTestData testData) if (expectedIndicator.Unit != null) { Assert.Equal(expectedIndicator.Unit, - EnumUtil.GetFromEnumLabel(actualIndicator.Unit)); + EnumUtil.GetFromEnumValue(actualIndicator.Unit)); } else { From 6961e5b404b81e01ece640290b29d8084ab42000 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Tue, 24 Sep 2024 11:36:58 +0100 Subject: [PATCH 10/80] EES-5494 Fix indicator units not parsing when processing into public API --- .../ProcessorTestData.cs | 1 + .../Models/MetaFileRow.cs | 8 ++++---- .../Repository/IndicatorMetaRepository.cs | 2 +- .../Commands/SeedDataCommand.cs | 2 +- .../Models/MetaFileRow.cs | 8 ++++---- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorTestData.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorTestData.cs index 63e97c881f5..91e1e06b77d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorTestData.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorTestData.cs @@ -329,6 +329,7 @@ public record ProcessorTestData Column = "sess_unauthorised_percent", Label = "Percentage of unauthorised sessions", DecimalPlaces = 2, + Unit = IndicatorUnit.Percent, DataSetVersionId = Guid.Empty }, ], diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Models/MetaFileRow.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Models/MetaFileRow.cs index fc1672fbbc9..81cc849750d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Models/MetaFileRow.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Models/MetaFileRow.cs @@ -13,11 +13,11 @@ public record MetaFileRow public string? IndicatorGrouping { get; init; } - private string? _indicatorUnit { get; init; } + public string? IndicatorUnit { get; init; } - public IndicatorUnit? IndicatorUnit => - _indicatorUnit is not null - ? EnumUtil.GetFromEnumValue(_indicatorUnit) + public IndicatorUnit? ParsedIndicatorUnit => + IndicatorUnit is not null + ? EnumUtil.GetFromEnumValue(IndicatorUnit) : null; public byte? IndicatorDp { get; init; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorMetaRepository.cs index 58266792db7..62578abefaf 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorMetaRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorMetaRepository.cs @@ -44,7 +44,7 @@ public async Task CreateIndicatorMetas( PublicId = SqidEncoder.Encode(id), Column = row.ColName, Label = row.Label, - Unit = row.IndicatorUnit, + Unit = row.ParsedIndicatorUnit, DecimalPlaces = row.IndicatorDp }; }) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Commands/SeedDataCommand.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Commands/SeedDataCommand.cs index 70dba2cb816..b4147499cda 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Commands/SeedDataCommand.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Commands/SeedDataCommand.cs @@ -407,7 +407,7 @@ private async Task CreateIndicatorMeta( PublicId = SqidEncoder.Encode(id), Column = row.ColName, Label = row.Label, - Unit = row.IndicatorUnit, + Unit = row.ParsedIndicatorUnit, DecimalPlaces = row.IndicatorDp }; }) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Models/MetaFileRow.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Models/MetaFileRow.cs index 3e80025c002..773cd52c76f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Models/MetaFileRow.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Scripts/Models/MetaFileRow.cs @@ -13,11 +13,11 @@ public record MetaFileRow public string? IndicatorGrouping { get; init; } - private string? _indicatorUnit { get; init; } + public string? IndicatorUnit { get; init; } - public IndicatorUnit? IndicatorUnit => - _indicatorUnit is not null - ? EnumUtil.GetFromEnumValue(_indicatorUnit) + public IndicatorUnit? ParsedIndicatorUnit => + IndicatorUnit is not null + ? EnumUtil.GetFromEnumValue(IndicatorUnit) : null; public byte? IndicatorDp { get; init; } From 01ff3cf204405df62d4fe9ffcf4d2203244eb1a1 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Fri, 27 Sep 2024 11:38:01 +0100 Subject: [PATCH 11/80] EES-5494 Fix content and schema types for `DownloadDataSetCsv` endpoint --- .../Controllers/DataSetsController.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs index f3e0b52f780..4b631a1f2f4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs @@ -262,10 +262,14 @@ public async Task> QueryData /// guarantees as the data set's JSON representation in other endpoints. /// [HttpGet("{dataSetId:guid}/csv")] - [Produces(MediaTypeNames.Text.Csv, MediaTypeNames.Application.Json)] - [SwaggerResponse(200, description: "The data set CSV file.", contentTypes: MediaTypeNames.Text.Csv)] - [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] - [SwaggerResponse(404, type: typeof(ProblemDetailsViewModel))] + [SwaggerResponse( + 200, + description: "The data set CSV file.", + type: typeof(string), + contentTypes: MediaTypeNames.Text.Csv + )] + [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel), contentTypes: MediaTypeNames.Application.Json)] + [SwaggerResponse(404, type: typeof(ProblemDetailsViewModel), contentTypes: MediaTypeNames.Application.Json)] public async Task DownloadDataSetCsv( [SwaggerParameter("The ID of the data set.")] Guid dataSetId, [SwaggerParameter("The version of the data set to use e.g. 2.0, 1.1, etc.")][FromQuery] string? dataSetVersion, From c6aa5a7a4f0d3346215cd1ceed3b9f87822d62ec Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Fri, 27 Sep 2024 11:40:29 +0100 Subject: [PATCH 12/80] EES-5494 Refactor `application/json` content types to use constant --- .../Controllers/DataSetVersionsController.cs | 7 ++++--- .../Controllers/DataSetsController.cs | 10 +++++----- .../Controllers/PublicationsController.cs | 7 ++++--- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs index 165c3236e1a..4b6b41df50c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs @@ -1,3 +1,4 @@ +using System.Net.Mime; using Asp.Versioning; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; @@ -24,7 +25,7 @@ public class DataSetVersionsController( /// List a data set’s versions. Only provides summary information of each version. /// [HttpGet] - [Produces("application/json")] + [Produces(MediaTypeNames.Application.Json)] [SwaggerResponse(200, "The paginated list of data set versions", type: typeof(DataSetVersionPaginatedListViewModel))] [SwaggerResponse(400, type: typeof(ValidationProblemViewModel))] [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] @@ -49,7 +50,7 @@ public async Task> ListDataSe /// Get a data set version's summary details. /// [HttpGet("{dataSetVersion}")] - [Produces("application/json")] + [Produces(MediaTypeNames.Application.Json)] [SwaggerResponse(200, "The requested data set version", type: typeof(DataSetVersionViewModel))] [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] [SwaggerResponse(404, type: typeof(ProblemDetailsViewModel))] @@ -73,7 +74,7 @@ public async Task> GetDataSetVersion( /// Lists the changes made by a data set version relative to the version prior to it. /// [HttpGet("{dataSetVersion}/changes")] - [Produces("application/json")] + [Produces(MediaTypeNames.Application.Json)] [SwaggerResponse(200, "The changes for this data set version", type: typeof(DataSetVersionChangesViewModel))] [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] [SwaggerResponse(404, type: typeof(ProblemDetailsViewModel))] diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs index 4b631a1f2f4..1f460ebbff1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs @@ -26,7 +26,7 @@ public class DataSetsController( /// Gets a specific data set’s summary details. /// [HttpGet("{dataSetId:guid}")] - [Produces("application/json")] + [Produces(MediaTypeNames.Application.Json)] [SwaggerResponse(200, "The requested data set summary", type: typeof(DataSetViewModel))] [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] [SwaggerResponse(404, type: typeof(ProblemDetailsViewModel))] @@ -48,7 +48,7 @@ public async Task> GetDataSet( /// Get the metadata about a data set. Use this to create data set queries. /// [HttpGet("{dataSetId:guid}/meta")] - [Produces("application/json")] + [Produces(MediaTypeNames.Application.Json)] [SwaggerResponse(200, "The requested data set version metadata", type: typeof(DataSetMetaViewModel))] [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] [SwaggerResponse(404, type: typeof(ProblemDetailsViewModel))] @@ -206,7 +206,7 @@ public async Task> GetDataSetMeta( /// - `location|REG|Asc` sorts by regions in ascending order /// [HttpGet("{dataSetId:guid}/query")] - [Produces("application/json")] + [Produces(MediaTypeNames.Application.Json)] [SwaggerResponse(200, "The paginated list of query results", type: typeof(DataSetQueryPaginatedResultsViewModel))] [SwaggerResponse(400, type: typeof(ValidationProblemViewModel))] [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] @@ -236,8 +236,8 @@ public async Task> QueryData /// and consequently can express more complex queries. /// [HttpPost("{dataSetId:guid}/query")] - [Consumes("application/json")] - [Produces("application/json")] + [Consumes(MediaTypeNames.Application.Json)] + [Produces(MediaTypeNames.Application.Json)] [SwaggerResponse(200, "The paginated list of query results", type: typeof(DataSetQueryPaginatedResultsViewModel))] [SwaggerResponse(400, type: typeof(ValidationProblemViewModel))] [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/PublicationsController.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/PublicationsController.cs index 5c792a61f90..d7abfce0a39 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/PublicationsController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/PublicationsController.cs @@ -1,3 +1,4 @@ +using System.Net.Mime; using Asp.Versioning; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.ViewModels; @@ -22,7 +23,7 @@ public class PublicationsController(IPublicationService publicationService, IDat /// Lists details about publications with data available for querying. /// [HttpGet] - [Produces("application/json")] + [Produces(MediaTypeNames.Application.Json)] [SwaggerResponse(200, "The paginated list of publications", type: typeof(PublicationPaginatedListViewModel))] [SwaggerResponse(400, type: typeof(ValidationProblemViewModel))] public async Task> ListPublications( @@ -45,7 +46,7 @@ public async Task> ListPublicati /// Get a specific publication's summary details. /// [HttpGet("{publicationId:guid}")] - [Produces("application/json")] + [Produces(MediaTypeNames.Application.Json)] [SwaggerResponse(200, "The requested publication summary", type: typeof(PublicationSummaryViewModel))] [SwaggerResponse(404, type: typeof(ProblemDetailsViewModel))] // add other responses @@ -67,7 +68,7 @@ public async Task> GetPublication( /// Lists summary details of all the data sets related to a publication. /// [HttpGet("{publicationId:guid}/data-sets")] - [Produces("application/json")] + [Produces(MediaTypeNames.Application.Json)] [SwaggerResponse(200, "The paginated list of data sets", type: typeof(DataSetPaginatedListViewModel))] [SwaggerResponse(400, type: typeof(ValidationProblemViewModel))] public async Task> ListPublicationDataSets( From bf5e833c108f12013cf98685a3b5e097743f47a2 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Fri, 27 Sep 2024 11:49:27 +0100 Subject: [PATCH 13/80] EES-5494 Add missing full stops to response descriptions --- .../Controllers/DataSetVersionsController.cs | 6 +++--- .../Controllers/DataSetsController.cs | 8 ++++---- .../Controllers/PublicationsController.cs | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs index 4b6b41df50c..dc46e2e74fb 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetVersionsController.cs @@ -26,7 +26,7 @@ public class DataSetVersionsController( /// [HttpGet] [Produces(MediaTypeNames.Application.Json)] - [SwaggerResponse(200, "The paginated list of data set versions", type: typeof(DataSetVersionPaginatedListViewModel))] + [SwaggerResponse(200, "The paginated list of data set versions.", type: typeof(DataSetVersionPaginatedListViewModel))] [SwaggerResponse(400, type: typeof(ValidationProblemViewModel))] [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] public async Task> ListDataSetVersions( @@ -51,7 +51,7 @@ public async Task> ListDataSe /// [HttpGet("{dataSetVersion}")] [Produces(MediaTypeNames.Application.Json)] - [SwaggerResponse(200, "The requested data set version", type: typeof(DataSetVersionViewModel))] + [SwaggerResponse(200, "The requested data set version.", type: typeof(DataSetVersionViewModel))] [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] [SwaggerResponse(404, type: typeof(ProblemDetailsViewModel))] public async Task> GetDataSetVersion( @@ -75,7 +75,7 @@ public async Task> GetDataSetVersion( /// [HttpGet("{dataSetVersion}/changes")] [Produces(MediaTypeNames.Application.Json)] - [SwaggerResponse(200, "The changes for this data set version", type: typeof(DataSetVersionChangesViewModel))] + [SwaggerResponse(200, "The changes for the data set version.", type: typeof(DataSetVersionChangesViewModel))] [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] [SwaggerResponse(404, type: typeof(ProblemDetailsViewModel))] public async Task> GetDataSetVersionChanges( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs index 1f460ebbff1..bae2847f8cf 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs @@ -27,7 +27,7 @@ public class DataSetsController( /// [HttpGet("{dataSetId:guid}")] [Produces(MediaTypeNames.Application.Json)] - [SwaggerResponse(200, "The requested data set summary", type: typeof(DataSetViewModel))] + [SwaggerResponse(200, "The requested data set summary.", type: typeof(DataSetViewModel))] [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] [SwaggerResponse(404, type: typeof(ProblemDetailsViewModel))] public async Task> GetDataSet( @@ -49,7 +49,7 @@ public async Task> GetDataSet( /// [HttpGet("{dataSetId:guid}/meta")] [Produces(MediaTypeNames.Application.Json)] - [SwaggerResponse(200, "The requested data set version metadata", type: typeof(DataSetMetaViewModel))] + [SwaggerResponse(200, "The requested data set version metadata.", type: typeof(DataSetMetaViewModel))] [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] [SwaggerResponse(404, type: typeof(ProblemDetailsViewModel))] public async Task> GetDataSetMeta( @@ -207,7 +207,7 @@ public async Task> GetDataSetMeta( /// [HttpGet("{dataSetId:guid}/query")] [Produces(MediaTypeNames.Application.Json)] - [SwaggerResponse(200, "The paginated list of query results", type: typeof(DataSetQueryPaginatedResultsViewModel))] + [SwaggerResponse(200, "The paginated list of query results.", type: typeof(DataSetQueryPaginatedResultsViewModel))] [SwaggerResponse(400, type: typeof(ValidationProblemViewModel))] [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] [SwaggerResponse(404, type: typeof(ProblemDetailsViewModel))] @@ -238,7 +238,7 @@ public async Task> QueryData [HttpPost("{dataSetId:guid}/query")] [Consumes(MediaTypeNames.Application.Json)] [Produces(MediaTypeNames.Application.Json)] - [SwaggerResponse(200, "The paginated list of query results", type: typeof(DataSetQueryPaginatedResultsViewModel))] + [SwaggerResponse(200, "The paginated list of query results.", type: typeof(DataSetQueryPaginatedResultsViewModel))] [SwaggerResponse(400, type: typeof(ValidationProblemViewModel))] [SwaggerResponse(403, type: typeof(ProblemDetailsViewModel))] [SwaggerResponse(404, type: typeof(ProblemDetailsViewModel))] diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/PublicationsController.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/PublicationsController.cs index d7abfce0a39..8ee6cdd0eb7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/PublicationsController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/PublicationsController.cs @@ -24,7 +24,7 @@ public class PublicationsController(IPublicationService publicationService, IDat /// [HttpGet] [Produces(MediaTypeNames.Application.Json)] - [SwaggerResponse(200, "The paginated list of publications", type: typeof(PublicationPaginatedListViewModel))] + [SwaggerResponse(200, "The paginated list of publications.", type: typeof(PublicationPaginatedListViewModel))] [SwaggerResponse(400, type: typeof(ValidationProblemViewModel))] public async Task> ListPublications( [FromQuery] PublicationListRequest request, @@ -47,7 +47,7 @@ public async Task> ListPublicati /// [HttpGet("{publicationId:guid}")] [Produces(MediaTypeNames.Application.Json)] - [SwaggerResponse(200, "The requested publication summary", type: typeof(PublicationSummaryViewModel))] + [SwaggerResponse(200, "The requested publication summary.", type: typeof(PublicationSummaryViewModel))] [SwaggerResponse(404, type: typeof(ProblemDetailsViewModel))] // add other responses public async Task> GetPublication( @@ -69,7 +69,7 @@ public async Task> GetPublication( /// [HttpGet("{publicationId:guid}/data-sets")] [Produces(MediaTypeNames.Application.Json)] - [SwaggerResponse(200, "The paginated list of data sets", type: typeof(DataSetPaginatedListViewModel))] + [SwaggerResponse(200, "The paginated list of data sets.", type: typeof(DataSetPaginatedListViewModel))] [SwaggerResponse(400, type: typeof(ValidationProblemViewModel))] public async Task> ListPublicationDataSets( [FromQuery] DataSetListRequest request, From c9fece13b5e03869795f488fe30c807ea96ae385 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Fri, 27 Sep 2024 15:35:11 +0100 Subject: [PATCH 14/80] EES-5494 Add explanation on why comments in JSON are allowed --- .../Startup.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs index 83962cca579..47e67154196 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Startup.cs @@ -136,6 +136,8 @@ public void ConfigureServices(IServiceCollection services) { options.JsonSerializerOptions.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull; options.JsonSerializerOptions.UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow; + // This allows comments to be left in JSON bodies so users can annotate + // their data set queries for debugging - do not remove! options.JsonSerializerOptions.ReadCommentHandling = JsonCommentHandling.Skip; // This must be false to allow `JsonExceptionResultFilter` to work correctly, From d87c6a4482d9d3f02e484a83a081612211217584 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Sun, 29 Sep 2024 11:37:32 +0100 Subject: [PATCH 15/80] EES-5494 Update doc for `QueryDataSetGet` endpoint --- .../Controllers/DataSetsController.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs index bae2847f8cf..ecd16d7cf40 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Controllers/DataSetsController.cs @@ -85,7 +85,7 @@ public async Task> GetDataSetMeta( /// /// The `indicators` query parameter is required and **at least one** indicator must be specified. /// - /// Each indicator should be a string containing the indicator ID e.g. `headcount`, `enrolments`. + /// Each indicator should be a string containing the indicator ID e.g. `4xbOu`, `8g1RI`. /// /// ## Filters /// @@ -140,7 +140,7 @@ public async Task> GetDataSetMeta( /// ### Examples /// /// - `LA|code|E08000019` matches any local authority with code `E08000019` - /// - `REG|id|abcde` matches any region with ID `abcde` + /// - `REG|id|6bQgZ` matches any region with ID `6bQgZ` /// - `SCH|urn|140821` matches any school with URN `140821` /// /// ## Time periods @@ -188,7 +188,7 @@ public async Task> GetDataSetMeta( /// Sorts are applied in the order they are provided and should be strings /// formatted like `{field}|{direction}` where: /// - /// - `field` is the name of the field to sort e.g. `time_period` + /// - `field` is the name of the field to sort e.g. `timePeriod` /// - `direction` is the direction to sort in e.g. ascending (`Asc`) or descending (`Desc`) /// /// The `field` can be one of the following: @@ -196,14 +196,16 @@ public async Task> GetDataSetMeta( /// - `timePeriod` to sort by time period /// - `geographicLevel` to sort by the geographic level of the data /// - `location|{level}` to sort by locations in a geographic level where `{level}` is the level code (e.g. `REG`, `LA`) - /// - A filter ID (e.g. `characteristic`, `school_type`) to sort by the options in that filter - /// - An indicator ID (e.g. `sess_authorised`, `enrolments`) to sort by the values in that indicator + /// - `filter|{id}` to sort by the options in a filter where `{id}` is the filter ID (e.g. `3RxWP`) + /// - `indicator|{id}` to sort by the values in a indicator where `{id}` is the indicator ID (e.g. `6VfPgZ`) /// /// ### Examples /// /// - `timePeriod|Desc` sorts by time period in descending order /// - `geographicLevel|Asc` sorts by geographic level in ascending order /// - `location|REG|Asc` sorts by regions in ascending order + /// - `filter|3RxWP|Desc` sorts by options in filter `3RxWP` in descending order + /// - `indicator|7a1dk|Asc` sorts by values in indicator `7a1dk` in ascending order /// [HttpGet("{dataSetId:guid}/query")] [Produces(MediaTypeNames.Application.Json)] From ed340de03fa9078c7f62d1252725fc37d9d54586 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Mon, 30 Sep 2024 14:42:53 +0100 Subject: [PATCH 16/80] EES-5524 Fix non-deterministic data set query sorting This fix appends an extra sort on the data's `id` column to ensure there is always a tie-break that can produce a deterministic order. --- .../DataSetsControllerGetQueryTests.cs | 194 +++++++++++++--- .../DataSetsControllerPostQueryTests.cs | 211 +++++++++++++++--- .../Services/DataSetQueryService.cs | 47 ++-- 3 files changed, 373 insertions(+), 79 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerGetQueryTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerGetQueryTests.cs index f55f62c132b..0a1287e3c12 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerGetQueryTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerGetQueryTests.cs @@ -1851,21 +1851,21 @@ public async Task SingleIndicator_Returns200_CorrectViewModel() var result = viewModel.Results[0]; Assert.Equal(2, result.Filters.Count); - Assert.Equal(AbsenceSchoolData.FilterNcYear10, result.Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal(AbsenceSchoolData.FilterNcYear4, result.Filters[AbsenceSchoolData.FilterNcYear]); Assert.Equal(AbsenceSchoolData.FilterSchoolTypeTotal, result.Filters[AbsenceSchoolData.FilterSchoolType]); Assert.Equal(GeographicLevel.LocalAuthority, result.GeographicLevel); Assert.Equal(3, result.Locations.Count); - Assert.Equal(AbsenceSchoolData.LocationLaKingstonUponThames, result.Locations["LA"]); + Assert.Equal(AbsenceSchoolData.LocationLaBarnsley, result.Locations["LA"]); Assert.Equal(AbsenceSchoolData.LocationNatEngland, result.Locations["NAT"]); - Assert.Equal(AbsenceSchoolData.LocationRegionOuterLondon, result.Locations["REG"]); + Assert.Equal(AbsenceSchoolData.LocationRegionYorkshire, result.Locations["REG"]); Assert.Equal(TimeIdentifier.AcademicYear, result.TimePeriod.Code); Assert.Equal("2022/2023", result.TimePeriod.Period); Assert.Single(result.Values); - Assert.Equal("4064499", result.Values[AbsenceSchoolData.IndicatorSessAuthorised]); + Assert.Equal("577798", result.Values[AbsenceSchoolData.IndicatorSessAuthorised]); } [Fact] @@ -2253,6 +2253,96 @@ public async Task AllFacets_MixOfConditions_Returns200() public class SortsTests(TestApplicationFactory testApp) : DataSetsControllerGetQueryTests(testApp) { + [Fact] + public async Task NoFields_SingleTimePeriod_Returns200() + { + var dataSetVersion = await SetupDefaultDataSetVersion(); + + var response = await QueryDataSet( + dataSetId: dataSetVersion.DataSetId, + indicators: [AbsenceSchoolData.IndicatorEnrolments], + queryParameters: new Dictionary + { + { "filters.eq", AbsenceSchoolData.FilterSchoolTypeTotal }, + { "geographicLevels.eq", "NAT" }, + { "timePeriods.eq", "2020/2021|AY" }, + } + ); + + var viewModel = response.AssertOk(useSystemJson: true); + + Assert.Equal(4, viewModel.Results.Count); + + Assert.All(viewModel.Results, result => + { + Assert.Equal(AbsenceSchoolData.FilterSchoolTypeTotal, result.Filters[AbsenceSchoolData.FilterSchoolType]); + Assert.Equal(GeographicLevel.Country, result.GeographicLevel); + Assert.Equal("2020/2021", result.TimePeriod.Period); + }); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[0].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("930365", viewModel.Results[0].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[1].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("390233", viewModel.Results[1].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear8, viewModel.Results[2].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("966035", viewModel.Results[2].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear10, viewModel.Results[3].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("687704", viewModel.Results[3].Values[AbsenceSchoolData.IndicatorEnrolments]); + } + + [Fact] + public async Task NoFields_MultipleTimePeriods_Returns200() + { + var dataSetVersion = await SetupDefaultDataSetVersion(); + + var response = await QueryDataSet( + dataSetId: dataSetVersion.DataSetId, + indicators: [AbsenceSchoolData.IndicatorEnrolments], + queryParameters: new Dictionary + { + { "filters.eq", AbsenceSchoolData.FilterSchoolTypePrimary }, + { "geographicLevels.eq", "NAT" } + } + ); + + var viewModel = response.AssertOk(useSystemJson: true); + + Assert.Equal(6, viewModel.Results.Count); + + Assert.All(viewModel.Results, result => + { + Assert.Equal(AbsenceSchoolData.FilterSchoolTypePrimary, result.Filters[AbsenceSchoolData.FilterSchoolType]); + Assert.Equal(GeographicLevel.Country, result.GeographicLevel); + }); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[0].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2022/2023", viewModel.Results[0].TimePeriod.Period); + Assert.Equal("654884", viewModel.Results[0].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[1].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2022/2023", viewModel.Results[1].TimePeriod.Period); + Assert.Equal("235647", viewModel.Results[1].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[2].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2021/2022", viewModel.Results[2].TimePeriod.Period); + Assert.Equal("611553", viewModel.Results[2].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[3].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2021/2022", viewModel.Results[3].TimePeriod.Period); + Assert.Equal("752711", viewModel.Results[3].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[4].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2020/2021", viewModel.Results[4].TimePeriod.Period); + Assert.Equal("233870", viewModel.Results[4].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[5].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2020/2021", viewModel.Results[5].TimePeriod.Period); + Assert.Equal("510682", viewModel.Results[5].Values[AbsenceSchoolData.IndicatorEnrolments]); + } + [Fact] public async Task SingleField_TimePeriodAsc_Returns200() { @@ -2261,10 +2351,10 @@ public async Task SingleField_TimePeriodAsc_Returns200() var response = await QueryDataSet( dataSetId: dataSetVersion.DataSetId, indicators: [AbsenceSchoolData.IndicatorEnrolments], - sorts: [$"timePeriod|Asc"], + sorts: ["timePeriod|Asc"], queryParameters: new Dictionary { - { "filters.eq", AbsenceSchoolData.FilterNcYear4 }, + { "filters.eq", AbsenceSchoolData.FilterSchoolTypePrimary }, { "geographicLevels.eq", "NAT" } } ); @@ -2273,14 +2363,35 @@ public async Task SingleField_TimePeriodAsc_Returns200() Assert.Equal(6, viewModel.Results.Count); - string[] expectedSequence = - [ - "2020/2021", - "2021/2022", - "2022/2023" - ]; + Assert.All(viewModel.Results, result => + { + Assert.Equal(AbsenceSchoolData.FilterSchoolTypePrimary, result.Filters[AbsenceSchoolData.FilterSchoolType]); + Assert.Equal(GeographicLevel.Country, result.GeographicLevel); + }); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[0].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2020/2021", viewModel.Results[0].TimePeriod.Period); + Assert.Equal("233870", viewModel.Results[0].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[1].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2020/2021", viewModel.Results[1].TimePeriod.Period); + Assert.Equal("510682", viewModel.Results[1].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[2].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2021/2022", viewModel.Results[2].TimePeriod.Period); + Assert.Equal("611553", viewModel.Results[2].Values[AbsenceSchoolData.IndicatorEnrolments]); - Assert.Equal(expectedSequence, GetSequence(viewModel.Results.Select(r => r.TimePeriod.Period))); + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[3].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2021/2022", viewModel.Results[3].TimePeriod.Period); + Assert.Equal("752711", viewModel.Results[3].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[4].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2022/2023", viewModel.Results[4].TimePeriod.Period); + Assert.Equal("654884", viewModel.Results[4].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[5].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2022/2023", viewModel.Results[5].TimePeriod.Period); + Assert.Equal("235647", viewModel.Results[5].Values[AbsenceSchoolData.IndicatorEnrolments]); } [Fact] @@ -2291,10 +2402,10 @@ public async Task SingleField_TimePeriodDesc_Returns200() var response = await QueryDataSet( dataSetId: dataSetVersion.DataSetId, indicators: [AbsenceSchoolData.IndicatorEnrolments], - sorts: [$"timePeriod|Desc"], + sorts: ["timePeriod|Desc"], queryParameters: new Dictionary { - { "filters.eq", AbsenceSchoolData.FilterNcYear4 }, + { "filters.eq", AbsenceSchoolData.FilterSchoolTypePrimary }, { "geographicLevels.eq", "NAT" } } ); @@ -2303,18 +2414,39 @@ public async Task SingleField_TimePeriodDesc_Returns200() Assert.Equal(6, viewModel.Results.Count); - string[] expectedSequence = - [ - "2022/2023", - "2021/2022", - "2020/2021" - ]; + Assert.All(viewModel.Results, result => + { + Assert.Equal(AbsenceSchoolData.FilterSchoolTypePrimary, result.Filters[AbsenceSchoolData.FilterSchoolType]); + Assert.Equal(GeographicLevel.Country, result.GeographicLevel); + }); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[0].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2022/2023", viewModel.Results[0].TimePeriod.Period); + Assert.Equal("654884", viewModel.Results[0].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[1].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2022/2023", viewModel.Results[1].TimePeriod.Period); + Assert.Equal("235647", viewModel.Results[1].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[2].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2021/2022", viewModel.Results[2].TimePeriod.Period); + Assert.Equal("611553", viewModel.Results[2].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[3].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2021/2022", viewModel.Results[3].TimePeriod.Period); + Assert.Equal("752711", viewModel.Results[3].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[4].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2020/2021", viewModel.Results[4].TimePeriod.Period); + Assert.Equal("233870", viewModel.Results[4].Values[AbsenceSchoolData.IndicatorEnrolments]); - Assert.Equal(expectedSequence, GetSequence(viewModel.Results.Select(r => r.TimePeriod.Period))); + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[5].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2020/2021", viewModel.Results[5].TimePeriod.Period); + Assert.Equal("510682", viewModel.Results[5].Values[AbsenceSchoolData.IndicatorEnrolments]); } [Fact] - public async Task SingleField_GeographicLevelAsc_Returns200() + public async Task SingleField_GeographicLevelAsc_Returns200_CorrectSequence() { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -2346,7 +2478,7 @@ public async Task SingleField_GeographicLevelAsc_Returns200() } [Fact] - public async Task SingleField_GeographicLevelDesc_Returns200() + public async Task SingleField_GeographicLevelDesc_Returns200_CorrectSequence() { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -2378,7 +2510,7 @@ public async Task SingleField_GeographicLevelDesc_Returns200() } [Fact] - public async Task SingleField_LocationAsc_Returns200() + public async Task SingleField_LocationAsc_Returns200_CorrectSequence() { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -2410,7 +2542,7 @@ public async Task SingleField_LocationAsc_Returns200() } [Fact] - public async Task SingleField_LocationDesc_Returns200() + public async Task SingleField_LocationDesc_Returns200_CorrectSequence() { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -2442,7 +2574,7 @@ public async Task SingleField_LocationDesc_Returns200() } [Fact] - public async Task SingleField_FilterAsc_Returns200() + public async Task SingleField_FilterAsc_Returns200_CorrectSequence() { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -2476,7 +2608,7 @@ public async Task SingleField_FilterAsc_Returns200() } [Fact] - public async Task SingleField_FilterDesc_Returns200() + public async Task SingleField_FilterDesc_Returns200_CorrectSequence() { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -2510,7 +2642,7 @@ public async Task SingleField_FilterDesc_Returns200() } [Fact] - public async Task SingleField_IndicatorAsc_Returns200() + public async Task SingleField_IndicatorAsc_Returns200_CorrectSequence() { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -2545,7 +2677,7 @@ public async Task SingleField_IndicatorAsc_Returns200() } [Fact] - public async Task SingleField_IndicatorDesc_Returns200() + public async Task SingleField_IndicatorDesc_Returns200_CorrectSequence() { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -2580,7 +2712,7 @@ public async Task SingleField_IndicatorDesc_Returns200() } [Fact] - public async Task MultipleFields_Returns200() + public async Task MultipleFields_Returns200_CorrectSequence() { var dataSetVersion = await SetupDefaultDataSetVersion(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerPostQueryTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerPostQueryTests.cs index 0fae45dd7ea..f1547e63fe4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerPostQueryTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetsControllerPostQueryTests.cs @@ -1820,21 +1820,21 @@ public async Task SingleIndicator_Returns200_CorrectViewModel() var result = viewModel.Results[0]; Assert.Equal(2, result.Filters.Count); - Assert.Equal(AbsenceSchoolData.FilterNcYear10, result.Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal(AbsenceSchoolData.FilterNcYear4, result.Filters[AbsenceSchoolData.FilterNcYear]); Assert.Equal(AbsenceSchoolData.FilterSchoolTypeTotal, result.Filters[AbsenceSchoolData.FilterSchoolType]); Assert.Equal(GeographicLevel.LocalAuthority, result.GeographicLevel); Assert.Equal(3, result.Locations.Count); - Assert.Equal(AbsenceSchoolData.LocationLaKingstonUponThames, result.Locations["LA"]); + Assert.Equal(AbsenceSchoolData.LocationLaBarnsley, result.Locations["LA"]); Assert.Equal(AbsenceSchoolData.LocationNatEngland, result.Locations["NAT"]); - Assert.Equal(AbsenceSchoolData.LocationRegionOuterLondon, result.Locations["REG"]); + Assert.Equal(AbsenceSchoolData.LocationRegionYorkshire, result.Locations["REG"]); Assert.Equal(TimeIdentifier.AcademicYear, result.TimePeriod.Code); Assert.Equal("2022/2023", result.TimePeriod.Period); Assert.Single(result.Values); - Assert.Equal("4064499", result.Values[AbsenceSchoolData.IndicatorSessAuthorised]); + Assert.Equal("577798", result.Values[AbsenceSchoolData.IndicatorSessAuthorised]); } [Fact] @@ -3019,6 +3019,117 @@ public async Task EquivalentCriteria_Returns200(IDataSetQueryCriteria criteria) public class SortsTests(TestApplicationFactory testApp) : DataSetsControllerPostQueryTests(testApp) { + [Fact] + public async Task NoFields_SingleTimePeriod_Returns200() + { + var dataSetVersion = await SetupDefaultDataSetVersion(); + + var response = await QueryDataSet( + dataSetId: dataSetVersion.DataSetId, + request: new DataSetQueryRequest + { + Indicators = [AbsenceSchoolData.IndicatorEnrolments], + Criteria = new DataSetQueryCriteriaFacets + { + Filters = new DataSetQueryCriteriaFilters + { + Eq = AbsenceSchoolData.FilterSchoolTypeTotal + }, + GeographicLevels = new DataSetQueryCriteriaGeographicLevels + { + Eq = "NAT" + }, + TimePeriods = new DataSetQueryCriteriaTimePeriods + { + Eq = new DataSetQueryTimePeriod { Code = "AY", Period = "2020/2021"} + } + } + } + ); + + var viewModel = response.AssertOk(useSystemJson: true); + + Assert.Equal(4, viewModel.Results.Count); + + Assert.All(viewModel.Results, result => + { + Assert.Equal(AbsenceSchoolData.FilterSchoolTypeTotal, result.Filters[AbsenceSchoolData.FilterSchoolType]); + Assert.Equal(GeographicLevel.Country, result.GeographicLevel); + Assert.Equal("2020/2021", result.TimePeriod.Period); + }); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[0].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("930365", viewModel.Results[0].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[1].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("390233", viewModel.Results[1].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear8, viewModel.Results[2].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("966035", viewModel.Results[2].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear10, viewModel.Results[3].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("687704", viewModel.Results[3].Values[AbsenceSchoolData.IndicatorEnrolments]); + } + + [Fact] + public async Task NoFields_MultipleTimePeriods_Returns200() + { + var dataSetVersion = await SetupDefaultDataSetVersion(); + + var response = await QueryDataSet( + dataSetId: dataSetVersion.DataSetId, + request: new DataSetQueryRequest + { + Indicators = [AbsenceSchoolData.IndicatorEnrolments], + Criteria = new DataSetQueryCriteriaFacets + { + Filters = new DataSetQueryCriteriaFilters + { + Eq = AbsenceSchoolData.FilterSchoolTypePrimary + }, + GeographicLevels = new DataSetQueryCriteriaGeographicLevels + { + Eq = "NAT" + }, + TimePeriods = new DataSetQueryCriteriaTimePeriods + { + In = + [ + new DataSetQueryTimePeriod { Code = "AY", Period = "2021/2022" }, + new DataSetQueryTimePeriod { Code = "AY", Period = "2022/2023" } + ] + } + } + } + ); + + var viewModel = response.AssertOk(useSystemJson: true); + + Assert.Equal(4, viewModel.Results.Count); + + Assert.All(viewModel.Results, result => + { + Assert.Equal(AbsenceSchoolData.FilterSchoolTypePrimary, result.Filters[AbsenceSchoolData.FilterSchoolType]); + Assert.Equal(GeographicLevel.Country, result.GeographicLevel); + }); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[0].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2022/2023", viewModel.Results[0].TimePeriod.Period); + Assert.Equal("654884", viewModel.Results[0].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[1].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2022/2023", viewModel.Results[1].TimePeriod.Period); + Assert.Equal("235647", viewModel.Results[1].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[2].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2021/2022", viewModel.Results[2].TimePeriod.Period); + Assert.Equal("611553", viewModel.Results[2].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[3].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2021/2022", viewModel.Results[3].TimePeriod.Period); + Assert.Equal("752711", viewModel.Results[3].Values[AbsenceSchoolData.IndicatorEnrolments]); + } + [Fact] public async Task SingleField_TimePeriodAsc_Returns200() { @@ -3034,7 +3145,7 @@ public async Task SingleField_TimePeriodAsc_Returns200() { Filters = new DataSetQueryCriteriaFilters { - Eq = AbsenceSchoolData.FilterNcYear4 + Eq = AbsenceSchoolData.FilterSchoolTypePrimary }, GeographicLevels = new DataSetQueryCriteriaGeographicLevels { @@ -3048,14 +3159,35 @@ public async Task SingleField_TimePeriodAsc_Returns200() Assert.Equal(6, viewModel.Results.Count); - string[] expectedSequence = - [ - "2020/2021", - "2021/2022", - "2022/2023" - ]; + Assert.All(viewModel.Results, result => + { + Assert.Equal(AbsenceSchoolData.FilterSchoolTypePrimary, result.Filters[AbsenceSchoolData.FilterSchoolType]); + Assert.Equal(GeographicLevel.Country, result.GeographicLevel); + }); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[0].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2020/2021", viewModel.Results[0].TimePeriod.Period); + Assert.Equal("233870", viewModel.Results[0].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[1].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2020/2021", viewModel.Results[1].TimePeriod.Period); + Assert.Equal("510682", viewModel.Results[1].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[2].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2021/2022", viewModel.Results[2].TimePeriod.Period); + Assert.Equal("611553", viewModel.Results[2].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[3].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2021/2022", viewModel.Results[3].TimePeriod.Period); + Assert.Equal("752711", viewModel.Results[3].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[4].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2022/2023", viewModel.Results[4].TimePeriod.Period); + Assert.Equal("654884", viewModel.Results[4].Values[AbsenceSchoolData.IndicatorEnrolments]); - Assert.Equal(expectedSequence, GetSequence(viewModel.Results.Select(r => r.TimePeriod.Period))); + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[5].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2022/2023", viewModel.Results[5].TimePeriod.Period); + Assert.Equal("235647", viewModel.Results[5].Values[AbsenceSchoolData.IndicatorEnrolments]); } [Fact] @@ -3073,7 +3205,7 @@ public async Task SingleField_TimePeriodDesc_Returns200() { Filters = new DataSetQueryCriteriaFilters { - Eq = AbsenceSchoolData.FilterNcYear4 + Eq = AbsenceSchoolData.FilterSchoolTypePrimary }, GeographicLevels = new DataSetQueryCriteriaGeographicLevels { @@ -3087,18 +3219,39 @@ public async Task SingleField_TimePeriodDesc_Returns200() Assert.Equal(6, viewModel.Results.Count); - string[] expectedSequence = - [ - "2022/2023", - "2021/2022", - "2020/2021" - ]; + Assert.All(viewModel.Results, result => + { + Assert.Equal(AbsenceSchoolData.FilterSchoolTypePrimary, result.Filters[AbsenceSchoolData.FilterSchoolType]); + Assert.Equal(GeographicLevel.Country, result.GeographicLevel); + }); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[0].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2022/2023", viewModel.Results[0].TimePeriod.Period); + Assert.Equal("654884", viewModel.Results[0].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[1].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2022/2023", viewModel.Results[1].TimePeriod.Period); + Assert.Equal("235647", viewModel.Results[1].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[2].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2021/2022", viewModel.Results[2].TimePeriod.Period); + Assert.Equal("611553", viewModel.Results[2].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[3].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2021/2022", viewModel.Results[3].TimePeriod.Period); + Assert.Equal("752711", viewModel.Results[3].Values[AbsenceSchoolData.IndicatorEnrolments]); + + Assert.Equal(AbsenceSchoolData.FilterNcYear4, viewModel.Results[4].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2020/2021", viewModel.Results[4].TimePeriod.Period); + Assert.Equal("233870", viewModel.Results[4].Values[AbsenceSchoolData.IndicatorEnrolments]); - Assert.Equal(expectedSequence, GetSequence(viewModel.Results.Select(r => r.TimePeriod.Period))); + Assert.Equal(AbsenceSchoolData.FilterNcYear6, viewModel.Results[5].Filters[AbsenceSchoolData.FilterNcYear]); + Assert.Equal("2020/2021", viewModel.Results[5].TimePeriod.Period); + Assert.Equal("510682", viewModel.Results[5].Values[AbsenceSchoolData.IndicatorEnrolments]); } [Fact] - public async Task SingleField_GeographicLevelAsc_Returns200() + public async Task SingleField_GeographicLevelAsc_Returns200_CorrectSequence() { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -3142,7 +3295,7 @@ public async Task SingleField_GeographicLevelAsc_Returns200() } [Fact] - public async Task SingleField_GeographicLevelDesc_Returns200() + public async Task SingleField_GeographicLevelDesc_Returns200_CorrectSequence() { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -3186,7 +3339,7 @@ public async Task SingleField_GeographicLevelDesc_Returns200() } [Fact] - public async Task SingleField_LocationAsc_Returns200() + public async Task SingleField_LocationAsc_Returns200_CorrectSequence() { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -3230,7 +3383,7 @@ public async Task SingleField_LocationAsc_Returns200() } [Fact] - public async Task SingleField_LocationDesc_Returns200() + public async Task SingleField_LocationDesc_Returns200_CorrectSequence() { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -3274,7 +3427,7 @@ public async Task SingleField_LocationDesc_Returns200() } [Fact] - public async Task SingleField_FilterAsc_Returns200() + public async Task SingleField_FilterAsc_Returns200_CorrectSequence() { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -3324,7 +3477,7 @@ public async Task SingleField_FilterAsc_Returns200() } [Fact] - public async Task SingleField_FilterDesc_Returns200() + public async Task SingleField_FilterDesc_Returns200_CorrectSequence() { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -3374,7 +3527,7 @@ public async Task SingleField_FilterDesc_Returns200() } [Fact] - public async Task SingleField_IndicatorAsc_Returns200() + public async Task SingleField_IndicatorAsc_Returns200_CorrectSequence() { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -3428,7 +3581,7 @@ public async Task SingleField_IndicatorAsc_Returns200() } [Fact] - public async Task SingleField_IndicatorDesc_Returns200() + public async Task SingleField_IndicatorDesc_Returns200_CorrectSequence() { var dataSetVersion = await SetupDefaultDataSetVersion(); @@ -3482,7 +3635,7 @@ public async Task SingleField_IndicatorDesc_Returns200() } [Fact] - public async Task MultipleFields_Returns200() + public async Task MultipleFields_Returns200_CorrectSequence() { var dataSetVersion = await SetupDefaultDataSetVersion(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Services/DataSetQueryService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Services/DataSetQueryService.cs index 2cae2a8b24c..3c9f212f958 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Services/DataSetQueryService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Services/DataSetQueryService.cs @@ -289,36 +289,45 @@ private static List GetSorts( ColumnMappings columnMappings, QueryState queryState) { - if (request.Sorts is null) - { - return [new Sort(Field: DataTable.Cols.TimePeriodId, Direction: SortDirection.Desc)]; - } - - var invalidSorts = new HashSet(); var sorts = new List(); - foreach (var sort in request.Sorts) + if (request.Sorts is not null) { - if (TryGetSortColumn(sort, columnMappings, out var column)) + var invalidSorts = new HashSet(); + + foreach (var sort in request.Sorts) { - sorts.Add(new Sort(Field: column, Direction: sort.ParsedDirection())); - continue; + if (TryGetSortColumn(sort, columnMappings, out var column)) + { + sorts.Add(new Sort(Field: column, Direction: sort.ParsedDirection())); + continue; + } + + invalidSorts.Add(sort); } - invalidSorts.Add(sort); + if (invalidSorts.Count != 0) + { + queryState.Errors.Add(new ErrorViewModel + { + Message = ValidationMessages.SortFieldsNotFound.Message, + Code = ValidationMessages.SortFieldsNotFound.Code, + Path = "sorts", + Detail = new NotFoundItemsErrorDetail(invalidSorts) + }); + } } - if (invalidSorts.Count != 0) + if (sorts.Count == 0) { - queryState.Errors.Add(new ErrorViewModel - { - Message = ValidationMessages.SortFieldsNotFound.Message, - Code = ValidationMessages.SortFieldsNotFound.Code, - Path = "sorts", - Detail = new NotFoundItemsErrorDetail(invalidSorts) - }); + // Default to sorting by time period in descending order if no sorts. + sorts.Add(new Sort(Field: DataTable.Ref().TimePeriodId, Direction: SortDirection.Desc)); } + // Need to append ID sort to ensure there is always a tie-break to ensure results are + // returned in a deterministic order (otherwise ordering can vary when queries are re-ran). + sorts.Add(new Sort(Field: DataTable.Ref().Id, Direction: SortDirection.Asc)); + return sorts; } From 0127c23bcbbd6c181b7c4d99ff1e3affb60fdcc1 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Mon, 30 Sep 2024 23:39:55 +0100 Subject: [PATCH 17/80] EES-5497 Update preview token usage guidance --- .../ReleaseApiDataSetPreviewTokenPage.tsx | 92 +++++++++++++++---- ...ReleaseApiDataSetPreviewTokenPage.test.tsx | 14 ++- .../src/components/CodeBlock.tsx | 4 +- .../public_api_dataset_preview_token.robot | 51 ++++------ 4 files changed, 102 insertions(+), 59 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx index 84429c544e9..fe3903cdeab 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx @@ -11,11 +11,14 @@ import previewTokenQueries from '@admin/queries/previewTokenQueries'; import apiDataSetQueries from '@admin/queries/apiDataSetQueries'; import previewTokenService from '@admin/services/previewTokenService'; import { useLastLocation } from '@admin/contexts/LastLocationContext'; +import CodeBlock from '@common/components/CodeBlock'; import LoadingSpinner from '@common/components/LoadingSpinner'; import Button from '@common/components/Button'; import CopyTextButton from '@common/components/CopyTextButton'; import FormattedDate from '@common/components/FormattedDate'; import ModalConfirm from '@common/components/ModalConfirm'; +import Tabs from '@common/components/Tabs'; +import TabsSection from '@common/components/TabsSection'; import ApiDataSetQuickStart from '@common/modules/data-catalogue/components/ApiDataSetQuickStart'; import { useQuery } from '@tanstack/react-query'; import { generatePath, useHistory, useParams } from 'react-router-dom'; @@ -75,6 +78,8 @@ export default function ReleaseApiDataSetPreviewTokenPage() { ); }; + const tokenExampleUrl = `${publicApiUrl}/api/v1.0/data-sets/${dataSet?.draftVersion?.id}`; + return ( <>
-
+
API data set preview token

{dataSet?.title}

{previewToken?.status === 'Active' ? ( <>

- Token expiry time:{' '} - - - {previewToken.expiry} - - + Reference: {previewToken.label}

-

Using this token

-

Step 1

-

TODO

-

Step 2

-

TODO else

-

Final checks

+ + +

+ The token expires:{' '} + + + {previewToken.expiry} + + +

+ +

Using the preview token

+ +

+ To use the preview token, add it to your request using a{' '} + Preview-Token header. +

+

+ The examples below illustrate how you can add the preview + token header to your code. +

+ + + + + {`curl -X GET -H "Preview-Token: ${previewToken.id}" \\ + ${tokenExampleUrl}`} + + + + + {`import requests + +url = "${tokenExampleUrl}" + +response = requests.get(url, headers={"Preview-Token": "${previewToken.id}"})`} + + + + {`library(httr) + +url <- "${tokenExampleUrl}" + +response <- GET(url, add_headers("Preview-Token" = "${previewToken.id}")) +`} + + + +

Revoking the preview token

+

- Please revoke the token as soon as you have finished checking - the API data set. + Once you have checked the API data set, it is recommended that + you revoke the preview token.

+ Revoke this token + } onConfirm={() => handleRevoke(previewToken.id)} > -

Are you sure you want to revoke this token?

+

Are you sure you want to revoke the preview token?

 +

- View API data set token log + View preview token log

+

API data set endpoints quick start

+ {dataSet?.draftVersion && ( { minute: '2-digit', })}`; - expect(screen.getByText('Token expiry time:')).toBeInTheDocument(); + expect(screen.getByText('The token expires:')).toBeInTheDocument(); expect(screen.getByText(expiry)).toBeInTheDocument(); expect( - screen.getByRole('heading', { name: 'Using this token' }), + screen.getByRole('heading', { name: 'Using the preview token' }), ).toBeInTheDocument(); expect(screen.getByLabelText('Preview token')).toHaveValue('token-id'); @@ -91,7 +91,7 @@ describe('ReleaseApiDataSetPreviewTokenPage', () => { ).toBeInTheDocument(); expect( - screen.getByRole('button', { name: 'Revoke this token' }), + screen.getByRole('button', { name: 'Revoke preview token' }), ).toBeInTheDocument(); expect( @@ -143,10 +143,14 @@ describe('ReleaseApiDataSetPreviewTokenPage', () => { await screen.findByText('API data set preview token'), ).toBeInTheDocument(); - await user.click(screen.getByRole('button', { name: 'Revoke this token' })); + await user.click( + screen.getByRole('button', { name: 'Revoke preview token' }), + ); expect( - await screen.findByText('Are you sure you want to revoke this token?'), + await screen.findByText( + 'Are you sure you want to revoke the preview token?', + ), ).toBeInTheDocument(); expect(previewTokenService.revokePreviewToken).not.toHaveBeenCalled(); diff --git a/src/explore-education-statistics-common/src/components/CodeBlock.tsx b/src/explore-education-statistics-common/src/components/CodeBlock.tsx index 56049c55b7a..fd370dbe435 100644 --- a/src/explore-education-statistics-common/src/components/CodeBlock.tsx +++ b/src/explore-education-statistics-common/src/components/CodeBlock.tsx @@ -1,17 +1,19 @@ import styles from '@common/components/CodeBlock.module.scss'; import useToggle from '@common/hooks/useToggle'; import React, { useEffect } from 'react'; +import bashLang from 'react-syntax-highlighter/dist/cjs/languages/hljs/bash'; import pythonLang from 'react-syntax-highlighter/dist/cjs/languages/hljs/python'; import rLang from 'react-syntax-highlighter/dist/cjs/languages/hljs/r'; import SyntaxHighlighter from 'react-syntax-highlighter/dist/cjs/light'; import Button from './Button'; +SyntaxHighlighter.registerLanguage('bash', bashLang); SyntaxHighlighter.registerLanguage('r', rLang); SyntaxHighlighter.registerLanguage('python', pythonLang); export interface CodeBlockProps { children: string; - language: 'python' | 'r'; + language: 'bash' | 'python' | 'r'; } export default function CodeBlock({ children, language }: CodeBlockProps) { diff --git a/tests/robot-tests/tests/public_api/public_api_dataset_preview_token.robot b/tests/robot-tests/tests/public_api/public_api_dataset_preview_token.robot index 9367c7882db..fdd4aa7f084 100644 --- a/tests/robot-tests/tests/public_api/public_api_dataset_preview_token.robot +++ b/tests/robot-tests/tests/public_api/public_api_dataset_preview_token.robot @@ -11,8 +11,6 @@ Suite Setup user signs in as bau1 Suite Teardown user closes the browser Test Setup fail test fast if required - - *** Variables *** ${PUBLICATION_NAME}= UI tests - Public API - Generate and Preview API token %{RUN_IDENTIFIER} ${RELEASE_NAME}= Academic year Q1 @@ -20,9 +18,6 @@ ${ACADEMIC_YEAR}= 3000 ${SUBJECT_NAME_1}= UI test subject 1 ${SUBJECT_NAME_2}= UI test subject 2 - - - *** Test Cases *** Create publication and release user selects dashboard theme and topic if possible @@ -94,20 +89,17 @@ Verify the contents inside the 'Draft API datasets' table user clicks link Back to API data sets user waits until h3 is visible Draft API data sets - user checks table column heading contains 1 1 Draft version xpath://table[@data-testid='draft-api-data-sets'] user checks table column heading contains 1 2 Name xpath://table[@data-testid='draft-api-data-sets'] user checks table column heading contains 1 3 Status xpath://table[@data-testid='draft-api-data-sets'] user checks table column heading contains 1 4 Actions xpath://table[@data-testid='draft-api-data-sets'] - user checks table cell contains 1 1 v1.0 xpath://table[@data-testid='draft-api-data-sets'] user checks table cell contains 1 3 Ready xpath://table[@data-testid='draft-api-data-sets'] user checks table cell contains 2 1 v1.0 xpath://table[@data-testid='draft-api-data-sets'] user checks table cell contains 2 3 Ready xpath://table[@data-testid='draft-api-data-sets'] - Click on 'View Details' link(First API dataset) user clicks link in table cell 1 4 View details xpath://table[@data-testid='draft-api-data-sets'] user waits until h3 is visible Draft version details @@ -121,7 +113,6 @@ User checks row data contents inside the 'Draft API datasets' summary table user checks contents inside the cell value National xpath://div[@data-testid="Geographic levels"]//dd[@data-testid="Geographic levels-value"] user checks contents inside the cell value 2012/13 xpath://div[@data-testid="Time periods"]//dd[@data-testid="Time periods-value"] - user checks contents inside the cell value Lower quartile annualised earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[1] user checks contents inside the cell value Median annualised earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[2] user checks contents inside the cell value Number of learners with earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[3] @@ -141,12 +132,10 @@ User checks row data contents inside the 'Draft API datasets' summary table user checks contents inside the cell value Number of years after achievement of learning aim xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[6] user checks contents inside the cell value Provision xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[7] - user checks contents inside the cell value Preview API data set xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[1]/a user checks contents inside the cell value View preview token log xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[2]/a user checks contents inside the cell value Remove draft version xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[3]/button - User clicks on 'Preview API dataset' link user clicks link containing text Preview API data set @@ -165,11 +154,11 @@ User creates API token through 'Generate API token' modal window user waits until h2 is visible ${SUBJECT_NAME_1} User revokes created API token - user clicks button Revoke this token - ${modal}= user waits until modal is visible Revoke this token + user clicks button Revoke preview token + ${modal}= user waits until modal is visible Revoke preview token user clicks button Confirm user waits until page finishes loading - user waits until modal is not visible Revoke this token %{WAIT_LONG} + user waits until modal is not visible Revoke preview token %{WAIT_LONG} user waits until page contains Generate API data set preview token User again clicks on 'Generate Token' @@ -187,11 +176,11 @@ User creates API token through 'Generate API token' modal window user waits until h2 is visible ${SUBJECT_NAME_1} User cancels to revoke created API token - user clicks button Revoke this token - ${modal}= user waits until modal is visible Revoke this token + user clicks button Revoke preview token + ${modal}= user waits until modal is visible Revoke preview token user clicks button Cancel user waits until page finishes loading - user waits until modal is not visible Revoke this token %{WAIT_LONG} + user waits until modal is not visible Revoke preview token %{WAIT_LONG} user waits until page contains API data set preview token user waits until h2 is visible ${SUBJECT_NAME_1} @@ -227,11 +216,11 @@ Verify the 'Active tokens' and 'Expired tokens' on preview token log page user checks table cell contains 2 4 Expired user clicks button in table cell 1 6 Revoke - ${modal}= user waits until modal is visible Revoke this token + ${modal}= user waits until modal is visible Revoke preview token user clicks button Confirm ${modal} user waits until page finishes loading user checks table cell contains 1 4 Expired - + User verifies the relevant fields on the 'View Log Details' page for the Active API token. user clicks link Generate preview token user clicks button Generate token @@ -249,18 +238,17 @@ User verifies the relevant fields on the 'View Log Details' page for the Active ${current_time_tomorrow}= get current datetime %Y-%m-%dT%H:%M:%S 1 Europe/London ${time_with_leading_zero}= format uk to local datetime ${current_time_tomorrow} %I:%M %p - + ${time_end}= format time without leading zero ${time_with_leading_zero} - user checks page contains - ... Token expiry time: tomorrow at ${time_end} + user checks page contains + ... The token expires: tomorrow at ${time_end} user checks page contains button Copy preview token - user checks page contains button Revoke this token - user checks page contains link View API data set token log + user checks page contains button Revoke preview token + user checks page contains link View preview token log user checks page contains API data set endpoints quick start - Add headline text block to Content page user clicks link Back to API data set details user navigates to content page ${PUBLICATION_NAME} @@ -290,7 +278,7 @@ Search with 1st API dataset User clicks on API dataset link user clicks link by index ${SUBJECT_NAME_1} user waits until page finishes loading - + user waits until h1 is visible ${SUBJECT_NAME_1} User checks relevant headings exist on API dataset details page @@ -314,25 +302,24 @@ User verifies the row headings and contents in 'Data set details' section user checks row headings within the api dataset section Notifications user checks contents inside the cell value Test theme css: #dataSetDetails [data-testid="Theme-value"] - user checks contents inside the cell value ${PUBLICATION_NAME} css:#dataSetDetails [data-testid="Publication-value"] + user checks contents inside the cell value ${PUBLICATION_NAME} css:#dataSetDetails [data-testid="Publication-value"] user checks contents inside the cell value Academic year Q1 3000/01 css:#dataSetDetails [data-testid="Release-value"] User checks contents inside the release type Accredited official statistics css:#dataSetDetails [data-testid="Release type-value"] > button user checks contents inside the cell value National css:#dataSetDetails [data-testid="Geographic levels-value"] - user checks contents inside the cell value Lower quartile annualised earnings css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(1) user checks contents inside the cell value Median annualised earnings css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(2) user checks contents inside the cell value Number of learners with earnings css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(3) - + user clicks button Show 1 more indicator css:#dataSetDetails [data-testid="Indicators-value"] - + user checks contents inside the cell value Upper quartile annualised earnings css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(4) user checks contents inside the cell value Cheese css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(1) user checks contents inside the cell value Colour css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(2) user checks contents inside the cell value Ethnicity group css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(3) - user clicks button Show 4 more filters css:#dataSetDetails [data-testid="Filters-value"] + user clicks button Show 4 more filters css:#dataSetDetails [data-testid="Filters-value"] user checks contents inside the cell value Gender css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(4) user checks contents inside the cell value Level of learning css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(5) @@ -352,8 +339,6 @@ User verifies the headings and contents in 'API version history' section user checks table cell contains 1 2 Academic year Q1 3000/01 xpath://section[@id="apiVersionHistory"] user checks table cell contains 1 3 Published xpath://section[@id="apiVersionHistory"] - - *** Keywords *** User checks contents inside the release type [Arguments] ${expected_text} ${locator} From 4d400bf79fa0a48a9a702405076f1790d3b3b8d6 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Wed, 2 Oct 2024 00:00:34 +0100 Subject: [PATCH 18/80] EES-5497 Fix various issues with `UrlContainer` and `CopyTextButton` This change: - Makes `label` required for `UrlContainer` so that the copy input is labelled correctly by the consumer. - Allowes inline styling to be correctly toggled using the `inline` prop for `UrlContainer`. - Changes `UrlContainer` and `CopyTextButton` to fill the parent's space as much as possible instead of being limited to 500px. - Fixed incorrect flex styling causing the input in `CopyTextButton` to not take up the correct amount of space. - Adds the GOV.UK focus style to the `UrlContainer` input. - Improves how `CopyTextButton` responds at mobile / tablet. - Updates `CopyTextButton` to show the `confirmMessage` in the button itself rather than in a separate line. --- .../ReleaseApiDataSetPreviewTokenPage.tsx | 5 +- .../src/components/CopyLinkModal.tsx | 23 ++-- .../src/components/CopyTextButton.module.scss | 27 ++++- .../src/components/CopyTextButton.tsx | 110 ++++++++---------- .../src/components/UrlContainer.module.scss | 8 +- .../src/components/UrlContainer.tsx | 19 +-- .../__tests__/CopyLinkModal.test.tsx | 29 ++++- ...TextButton.tsx => CopyTextButton.test.tsx} | 9 +- .../src/styles/utils/_flex.scss | 8 ++ .../components/DataSetFileUsage.tsx | 12 +- .../__tests__/DataSetFileUsage.test.tsx | 2 +- 11 files changed, 142 insertions(+), 110 deletions(-) rename src/explore-education-statistics-common/src/components/__tests__/{CopyTextButton.tsx => CopyTextButton.test.tsx} (78%) diff --git a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx index fe3903cdeab..57f1afa833a 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx @@ -109,9 +109,8 @@ export default function ReleaseApiDataSetPreviewTokenPage() { diff --git a/src/explore-education-statistics-common/src/components/CopyLinkModal.tsx b/src/explore-education-statistics-common/src/components/CopyLinkModal.tsx index a03cef6bd84..d2132b01f3f 100644 --- a/src/explore-education-statistics-common/src/components/CopyLinkModal.tsx +++ b/src/explore-education-statistics-common/src/components/CopyLinkModal.tsx @@ -2,24 +2,15 @@ import Button from '@common/components/Button'; import LinkIcon from '@common/components/LinkIcon'; import Modal from '@common/components/Modal'; import VisuallyHidden from '@common/components/VisuallyHidden'; -import CopyTextButton, { - CopyTextButtonProps, -} from '@common/components/CopyTextButton'; -import { OmitStrict } from '@common/types'; +import CopyTextButton from '@common/components/CopyTextButton'; import React from 'react'; -interface Props extends OmitStrict { +interface Props { buttonClassName?: string; url: string; } -export default function CopyLinkModal({ - buttonClassName, - className, - confirmMessage = 'Link copied to the clipboard.', - inlineButton = true, - url, -}: Props) { +export default function CopyLinkModal({ buttonClassName, url }: Props) { return ( diff --git a/src/explore-education-statistics-common/src/components/CopyTextButton.module.scss b/src/explore-education-statistics-common/src/components/CopyTextButton.module.scss index b4f7cda82a3..cfb2592bc40 100644 --- a/src/explore-education-statistics-common/src/components/CopyTextButton.module.scss +++ b/src/explore-education-statistics-common/src/components/CopyTextButton.module.scss @@ -1,13 +1,28 @@ @import '~govuk-frontend/dist/govuk/base'; -.message { - height: 2rem; - margin: govuk-spacing(2) 0; +.container { + margin-bottom: govuk-spacing(4); + + @include govuk-media-query($until: tablet) { + display: block; + } } -.container { +.urlContainer { + @include govuk-media-query($until: tablet) { + margin-bottom: govuk-spacing(2); + } +} + +.button { + display: block; + @include govuk-media-query($from: tablet) { - width: 500px; - max-width: 100%; + box-shadow: 0 0; + display: inline-block; + flex-shrink: 1; + margin-bottom: 0; + white-space: nowrap; + width: auto; } } diff --git a/src/explore-education-statistics-common/src/components/CopyTextButton.tsx b/src/explore-education-statistics-common/src/components/CopyTextButton.tsx index de45e141bee..f88754f1ddd 100644 --- a/src/explore-education-statistics-common/src/components/CopyTextButton.tsx +++ b/src/explore-education-statistics-common/src/components/CopyTextButton.tsx @@ -1,16 +1,18 @@ import Button from '@common/components/Button'; import styles from '@common/components/CopyTextButton.module.scss'; +import ScreenReaderMessage from '@common/components/ScreenReaderMessage'; import UrlContainer from '@common/components/UrlContainer'; import useToggle from '@common/hooks/useToggle'; import classNames from 'classnames'; -import React, { ReactNode } from 'react'; +import React, { ReactNode, useEffect } from 'react'; export interface CopyTextButtonProps { buttonText?: string | ReactNode; className?: string; - confirmMessage?: string; - inlineButton?: boolean; - label?: string; + confirmText?: string; + id: string; + inline?: boolean; + label: string; labelHidden?: boolean; text: string; } @@ -18,74 +20,56 @@ export interface CopyTextButtonProps { export default function CopyTextButton({ buttonText = 'Copy', className, - confirmMessage = 'Text copied to the clipboard.', - inlineButton = true, + confirmText = 'Copied', + id, + inline = true, label, - labelHidden = true, + labelHidden, text, }: CopyTextButtonProps) { const [copied, toggleCopied] = useToggle(false); + useEffect(() => { + const resetTimeout = setTimeout(toggleCopied.off, 5000); + + return () => { + if (copied) { + clearTimeout(resetTimeout); + } + }; + }, [copied, toggleCopied]); + return ( -
-
+ + + - {!inlineButton && ( - - )} -
-
- {inlineButton && ( - - )} -
- ); -} + {copied ? confirmText : buttonText} + -function Message({ - className, - copied, - confirmMessage, -}: { - className?: string; - copied: boolean; - confirmMessage?: string; -}) { - return ( -
- {copied && confirmMessage} +
); } diff --git a/src/explore-education-statistics-common/src/components/UrlContainer.module.scss b/src/explore-education-statistics-common/src/components/UrlContainer.module.scss index 0b3b8f6367c..513eb16735d 100644 --- a/src/explore-education-statistics-common/src/components/UrlContainer.module.scss +++ b/src/explore-education-statistics-common/src/components/UrlContainer.module.scss @@ -1,11 +1,17 @@ @import '~govuk-frontend/dist/govuk/base'; +@import '@common/styles/mixins/focus'; .url { background: govuk-colour('light-grey'); border: 0; display: inline-block; - flex: 1; + flex-grow: 1; padding: govuk-spacing(2) govuk-spacing(4); + text-overflow: ellipsis; user-select: all; white-space: pre-wrap; + + &:focus { + @include focused-input; + } } diff --git a/src/explore-education-statistics-common/src/components/UrlContainer.tsx b/src/explore-education-statistics-common/src/components/UrlContainer.tsx index 3d07bf387b1..1e6a4a7549c 100644 --- a/src/explore-education-statistics-common/src/components/UrlContainer.tsx +++ b/src/explore-education-statistics-common/src/components/UrlContainer.tsx @@ -5,9 +5,9 @@ import React, { ReactNode } from 'react'; interface Props { className?: string; id: string; + inline?: boolean; label?: string | ReactNode; labelHidden?: boolean; - widthLimited?: boolean; testId?: string; url: string; } @@ -15,17 +15,17 @@ interface Props { export default function UrlContainer({ className, id, + inline = true, label = 'URL', labelHidden, - widthLimited, testId = id, url, }: Props) { return (
e.target.select()} + id={id} readOnly + type="text" + value={url} + onFocus={e => e.target.select()} />
); diff --git a/src/explore-education-statistics-common/src/components/__tests__/CopyLinkModal.test.tsx b/src/explore-education-statistics-common/src/components/__tests__/CopyLinkModal.test.tsx index 517f07f94c7..11222087348 100644 --- a/src/explore-education-statistics-common/src/components/__tests__/CopyLinkModal.test.tsx +++ b/src/explore-education-statistics-common/src/components/__tests__/CopyLinkModal.test.tsx @@ -1,10 +1,10 @@ import CopyLinkModal from '@common/components/CopyLinkModal'; import render from '@common-test/render'; import React from 'react'; -import { screen, within } from '@testing-library/react'; +import { screen, waitFor, within } from '@testing-library/react'; describe('CopyLinkModal', () => { - const testUrl = 'http://test.com/1#test-heading'; + const testUrl = 'https://test.com/1#test-heading'; test('renders the copy link button', () => { render(); @@ -22,10 +22,33 @@ describe('CopyLinkModal', () => { ); const modal = within(screen.getByRole('dialog')); + expect( modal.getByRole('heading', { name: 'Copy link to the clipboard' }), ).toBeInTheDocument(); expect(modal.getByLabelText('URL')).toHaveValue(testUrl); - expect(modal.getByRole('button', { name: 'Copy' })).toBeInTheDocument(); + expect( + modal.getByRole('button', { name: 'Copy link' }), + ).toBeInTheDocument(); + }); + + test('copies the text to the clipboard and shows a message when the button is clicked', async () => { + const { user } = render(); + + await user.click( + screen.getByRole('button', { name: 'Copy link to the clipboard' }), + ); + + const modal = within(screen.getByRole('dialog')); + + await user.click(modal.getByRole('button', { name: 'Copy link' })); + + await waitFor(() => { + expect(screen.getByText('Link copied')).toBeInTheDocument(); + }); + + const copiedText = await window.navigator.clipboard.readText(); + + expect(copiedText).toEqual(testUrl); }); }); diff --git a/src/explore-education-statistics-common/src/components/__tests__/CopyTextButton.tsx b/src/explore-education-statistics-common/src/components/__tests__/CopyTextButton.test.tsx similarity index 78% rename from src/explore-education-statistics-common/src/components/__tests__/CopyTextButton.tsx rename to src/explore-education-statistics-common/src/components/__tests__/CopyTextButton.test.tsx index 9b79f0915d8..d1a3a9c47bd 100644 --- a/src/explore-education-statistics-common/src/components/__tests__/CopyTextButton.tsx +++ b/src/explore-education-statistics-common/src/components/__tests__/CopyTextButton.test.tsx @@ -7,17 +7,18 @@ describe('CopyTextButton', () => { const testText = 'http://test.com/1#test-heading'; test('copies the text to the clipboard and shows a message when the button is clicked', async () => { - const { user } = render(); + const { user } = render( + , + ); await user.click(screen.getByRole('button', { name: 'Copy' })); await waitFor(() => { - expect( - screen.getByText('Text copied to the clipboard.'), - ).toBeInTheDocument(); + expect(screen.getByText('Copied')).toBeInTheDocument(); }); const copiedText = await window.navigator.clipboard.readText(); + expect(copiedText).toEqual(testText); }); }); diff --git a/src/explore-education-statistics-common/src/styles/utils/_flex.scss b/src/explore-education-statistics-common/src/styles/utils/_flex.scss index 4b3947ab9b2..b83d67ca385 100644 --- a/src/explore-education-statistics-common/src/styles/utils/_flex.scss +++ b/src/explore-education-statistics-common/src/styles/utils/_flex.scss @@ -2,6 +2,14 @@ display: flex; } +.dfe-flex-direction--column { + flex-direction: column; +} + +.dfe-flex-direction--row { + flex-direction: row; +} + .dfe-flex-wrap { flex-wrap: wrap; } diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/DataSetFileUsage.tsx b/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/DataSetFileUsage.tsx index b29feefe132..4607f08becf 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/DataSetFileUsage.tsx +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/DataSetFileUsage.tsx @@ -24,7 +24,7 @@ export default function DataSetFileUsage({ tableToolLink, onDownload, }: Props) { - const downloadLink = new URL( + const downloadUrl = new URL( `/data-catalogue/data-set/${dataSetFileId}/csv`, process.env.PUBLIC_URL, ).href; @@ -71,8 +71,10 @@ export default function DataSetFileUsage({

@@ -85,13 +87,13 @@ export default function DataSetFileUsage({ {`import pandas as pd -pd.read_csv("${downloadLink}")`} +pd.read_csv("${downloadUrl}")`}
R
- {`read.csv("${downloadLink}")`} + {`read.csv("${downloadUrl}")`} diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/__tests__/DataSetFileUsage.test.tsx b/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/__tests__/DataSetFileUsage.test.tsx index b6cc3c86b4e..dd0f34c5e7c 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/__tests__/DataSetFileUsage.test.tsx +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/__tests__/DataSetFileUsage.test.tsx @@ -95,6 +95,6 @@ describe('DataSetFileUsage', () => { />, ); - expect(screen.getByTestId('copy-link-url')).toBeInTheDocument(); + expect(screen.getByTestId('copy-download-url')).toBeInTheDocument(); }); }); From 739d71e68a20c16efd7f7701b29d52a0a3b480de Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Wed, 2 Oct 2024 00:01:55 +0100 Subject: [PATCH 19/80] EES-5497 Update `CopyLinkModal` to be wider so show more of the link --- .../src/components/CopyLinkModal.module.scss | 9 +++++++++ .../src/components/CopyLinkModal.tsx | 2 ++ 2 files changed, 11 insertions(+) create mode 100644 src/explore-education-statistics-common/src/components/CopyLinkModal.module.scss diff --git a/src/explore-education-statistics-common/src/components/CopyLinkModal.module.scss b/src/explore-education-statistics-common/src/components/CopyLinkModal.module.scss new file mode 100644 index 00000000000..54d5faa6bed --- /dev/null +++ b/src/explore-education-statistics-common/src/components/CopyLinkModal.module.scss @@ -0,0 +1,9 @@ +@import '~govuk-frontend/dist/govuk/base'; + +.copyLink { + margin-bottom: govuk-spacing(4); + + @include govuk-media-query($from: desktop) { + min-width: map-get($govuk-breakpoints, 'tablet'); + } +} diff --git a/src/explore-education-statistics-common/src/components/CopyLinkModal.tsx b/src/explore-education-statistics-common/src/components/CopyLinkModal.tsx index d2132b01f3f..47c5611e41f 100644 --- a/src/explore-education-statistics-common/src/components/CopyLinkModal.tsx +++ b/src/explore-education-statistics-common/src/components/CopyLinkModal.tsx @@ -4,6 +4,7 @@ import Modal from '@common/components/Modal'; import VisuallyHidden from '@common/components/VisuallyHidden'; import CopyTextButton from '@common/components/CopyTextButton'; import React from 'react'; +import styles from './CopyLinkModal.module.scss'; interface Props { buttonClassName?: string; @@ -24,6 +25,7 @@ export default function CopyLinkModal({ buttonClassName, url }: Props) { > Date: Thu, 3 Oct 2024 11:10:58 +0100 Subject: [PATCH 20/80] EES-5553 Fix `CodeBlock` announcing default copy text for screen readers --- .../src/components/CodeBlock.tsx | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/explore-education-statistics-common/src/components/CodeBlock.tsx b/src/explore-education-statistics-common/src/components/CodeBlock.tsx index fd370dbe435..5a5972c8fb0 100644 --- a/src/explore-education-statistics-common/src/components/CodeBlock.tsx +++ b/src/explore-education-statistics-common/src/components/CodeBlock.tsx @@ -1,4 +1,5 @@ import styles from '@common/components/CodeBlock.module.scss'; +import ScreenReaderMessage from '@common/components/ScreenReaderMessage'; import useToggle from '@common/hooks/useToggle'; import React, { useEffect } from 'react'; import bashLang from 'react-syntax-highlighter/dist/cjs/languages/hljs/bash'; @@ -13,10 +14,17 @@ SyntaxHighlighter.registerLanguage('python', pythonLang); export interface CodeBlockProps { children: string; + copyConfirmText?: string; + copyText?: string; language: 'bash' | 'python' | 'r'; } -export default function CodeBlock({ children, language }: CodeBlockProps) { +export default function CodeBlock({ + children, + copyConfirmText = 'Code copied', + copyText = 'Copy code', + language, +}: CodeBlockProps) { const [copied, toggleCopied] = useToggle(false); useEffect(() => { @@ -38,9 +46,11 @@ export default function CodeBlock({ children, language }: CodeBlockProps) { toggleCopied.on(); }} > - {copied ? 'Code copied' : 'Copy code'} + {copied ? copyConfirmText : copyText} + + Date: Wed, 2 Oct 2024 03:26:03 +0100 Subject: [PATCH 21/80] EES-5497 Update preview token wording and relevant Robot tests --- .../data/ReleaseApiDataSetPreviewPage.tsx | 4 +- .../ReleaseApiDataSetPreviewTokenLogPage.tsx | 7 +- .../ReleaseApiDataSetPreviewPage.test.tsx | 6 +- ...easeApiDataSetPreviewTokenLogPage.test.tsx | 4 +- ...ReleaseApiDataSetPreviewTokenPage.test.tsx | 2 +- .../ApiDataSetPreviewTokenCreateForm.tsx | 2 +- .../tests/libs/public-api-common.robot | 6 +- ...ot => public_api_cancel_and_removal.robot} | 4 +- ...n.robot => public_api_preview_token.robot} | 81 ++++++++++--------- .../public_api_resolve_mapping_statuses.robot | 6 +- .../public_api/public_api_restricted.robot | 16 ++-- 11 files changed, 74 insertions(+), 64 deletions(-) rename tests/robot-tests/tests/public_api/{public_api_dataset_cancel_and_removal.robot => public_api_cancel_and_removal.robot} (98%) rename tests/robot-tests/tests/public_api/{public_api_dataset_preview_token.robot => public_api_preview_token.robot} (88%) diff --git a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewPage.tsx b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewPage.tsx index 66677b88567..b9ea9976c85 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewPage.tsx @@ -96,13 +96,13 @@ export default function ReleaseApiDataSetPreviewPage() {

- Generate token + Generate preview token } > diff --git a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenLogPage.tsx b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenLogPage.tsx index d82dab9aef6..4771d79f6de 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenLogPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenLogPage.tsx @@ -126,7 +126,7 @@ export default function ReleaseApiDataSetPreviewTokenLogPage() { for {token.label} Revoke @@ -135,7 +135,10 @@ export default function ReleaseApiDataSetPreviewTokenLogPage() { } onConfirm={() => handleRevoke(token.id)} > -

Are you sure you want to revoke this token?

+

+ Are you sure you want to revoke this preview + token? +

)} diff --git a/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetPreviewPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetPreviewPage.test.tsx index 205a69bdef2..bc24e309927 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetPreviewPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetPreviewPage.test.tsx @@ -72,7 +72,7 @@ describe('ReleaseApiDataSetPreviewPage', () => { screen.getByRole('heading', { name: 'Data set title' }), ).toBeInTheDocument(); expect( - screen.getByRole('button', { name: 'Generate token' }), + screen.getByRole('button', { name: 'Generate preview token' }), ).toBeInTheDocument(); }); @@ -87,7 +87,9 @@ describe('ReleaseApiDataSetPreviewPage', () => { await screen.findByText('Generate API data set preview token'), ).toBeInTheDocument(); - await user.click(screen.getByRole('button', { name: 'Generate token' })); + await user.click( + screen.getByRole('button', { name: 'Generate preview token' }), + ); expect(await screen.findByText('Token name')).toBeInTheDocument(); diff --git a/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetPreviewTokenLogPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetPreviewTokenLogPage.test.tsx index 24bb996e4e2..6a713dad0ba 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetPreviewTokenLogPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetPreviewTokenLogPage.test.tsx @@ -197,7 +197,9 @@ describe('ReleaseApiDataSetPreviewTokenLogPage', () => { ); expect( - await screen.findByText('Are you sure you want to revoke this token?'), + await screen.findByText( + 'Are you sure you want to revoke this preview token?', + ), ).toBeInTheDocument(); expect(previewTokenService.revokePreviewToken).not.toHaveBeenCalled(); diff --git a/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetPreviewTokenPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetPreviewTokenPage.test.tsx index 56cfdfdcfce..24c139fc83c 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetPreviewTokenPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetPreviewTokenPage.test.tsx @@ -95,7 +95,7 @@ describe('ReleaseApiDataSetPreviewTokenPage', () => { ).toBeInTheDocument(); expect( - screen.getByRole('link', { name: 'View API data set token log' }), + screen.getByRole('link', { name: 'View preview token log' }), ).toBeInTheDocument(); expect( diff --git a/src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetPreviewTokenCreateForm.tsx b/src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetPreviewTokenCreateForm.tsx index c666812798e..bb7cfa1714e 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetPreviewTokenCreateForm.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/components/ApiDataSetPreviewTokenCreateForm.tsx @@ -76,7 +76,7 @@ export default function ApiDataSetPreviewTokenCreateForm({ inline loading={formState.isSubmitting} size="sm" - text="Creating new API data set preview token" + text="Creating new preview token" /> diff --git a/tests/robot-tests/tests/libs/public-api-common.robot b/tests/robot-tests/tests/libs/public-api-common.robot index 780e441ff0f..b719d79dffd 100644 --- a/tests/robot-tests/tests/libs/public-api-common.robot +++ b/tests/robot-tests/tests/libs/public-api-common.robot @@ -46,19 +46,19 @@ verify status of API Datasets user waits for caches to expire ${status_value}= get text xpath:(//div[@data-testid="Status"]//dd[@data-testid="Status-value"]//strong)[2] should be equal as strings ${status_value} ${expected_status} - + user checks status in Draft version table [Arguments] ${text} ${expected_status} user waits for caches to expire ${status_value}= get text xpath:(//div[@data-testid="Status"]//dd[@data-testid="${text}"]//strong)[2] should be equal as strings ${status_value} ${expected_status} -user checks row headings within the api dataset section +user checks row headings within the api data set section [Arguments] ${text} ${parent}=#dataSetDetails user waits until page contains element css:${PARENT} [data-testid="${text}"] > dt user gets accordion header button element - [Arguments] ${heading_text} ${parent}=css:#dataSetDetails + [Arguments] ${heading_text} ${parent}=css:#dataSetDetails ${button}= get child element ${parent} css:.[data-testid="Release"] > dt [Return] ${button} diff --git a/tests/robot-tests/tests/public_api/public_api_dataset_cancel_and_removal.robot b/tests/robot-tests/tests/public_api/public_api_cancel_and_removal.robot similarity index 98% rename from tests/robot-tests/tests/public_api/public_api_dataset_cancel_and_removal.robot rename to tests/robot-tests/tests/public_api/public_api_cancel_and_removal.robot index 722fd37d616..5675746d6fe 100644 --- a/tests/robot-tests/tests/public_api/public_api_dataset_cancel_and_removal.robot +++ b/tests/robot-tests/tests/public_api/public_api_cancel_and_removal.robot @@ -14,7 +14,7 @@ Test Setup fail test fast if required *** Variables *** -${PUBLICATION_NAME}= UI tests - public api cancel and removal %{RUN_IDENTIFIER} +${PUBLICATION_NAME}= UI tests - Public API - cancel and removal %{RUN_IDENTIFIER} ${RELEASE_NAME}= Financial year 3000-01 ${SUBJECT_NAME_1}= UI test subject 1 ${SUBJECT_NAME_2}= UI test subject 2 @@ -117,7 +117,7 @@ Verify the contents inside the 'Draft API datasets' table Remove draft API dataset user clicks button in table cell 1 4 Remove draft xpath://table[@data-testid='draft-api-data-sets'] - + ${modal}= user waits until modal is visible Remove this draft API data set version user clicks button Remove this API data set version diff --git a/tests/robot-tests/tests/public_api/public_api_dataset_preview_token.robot b/tests/robot-tests/tests/public_api/public_api_preview_token.robot similarity index 88% rename from tests/robot-tests/tests/public_api/public_api_dataset_preview_token.robot rename to tests/robot-tests/tests/public_api/public_api_preview_token.robot index fdd4aa7f084..5f1ff219c08 100644 --- a/tests/robot-tests/tests/public_api/public_api_dataset_preview_token.robot +++ b/tests/robot-tests/tests/public_api/public_api_preview_token.robot @@ -12,11 +12,12 @@ Suite Teardown user closes the browser Test Setup fail test fast if required *** Variables *** -${PUBLICATION_NAME}= UI tests - Public API - Generate and Preview API token %{RUN_IDENTIFIER} +${PUBLICATION_NAME}= UI tests - Public API - preview token %{RUN_IDENTIFIER} ${RELEASE_NAME}= Academic year Q1 ${ACADEMIC_YEAR}= 3000 ${SUBJECT_NAME_1}= UI test subject 1 ${SUBJECT_NAME_2}= UI test subject 2 +${PREVIEW_TOKEN_NAME}= Test token *** Test Cases *** Create publication and release @@ -136,24 +137,24 @@ User checks row data contents inside the 'Draft API datasets' summary table user checks contents inside the cell value View preview token log xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[2]/a user checks contents inside the cell value Remove draft version xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[3]/button -User clicks on 'Preview API dataset' link +User clicks on 'Preview API data set' link user clicks link containing text Preview API data set -User clicks on 'Generate Token' - user clicks button Generate token +User clicks on 'Generate preview token' + user clicks button Generate preview token -User creates API token through 'Generate API token' modal window - ${modal}= user waits until modal is visible Generate API token - user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] API Token +User creates preview token through 'Generate preview token' modal window + ${modal}= user waits until modal is visible Generate preview token + user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] ${PREVIEW_TOKEN_NAME} user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] user clicks button Continue user waits until page finishes loading - user waits until modal is not visible Generate API token %{WAIT_LONG} + user waits until modal is not visible Generate preview token %{WAIT_LONG} user waits until page contains API data set preview token user waits until h2 is visible ${SUBJECT_NAME_1} -User revokes created API token +User revokes preview token user clicks button Revoke preview token ${modal}= user waits until modal is visible Revoke preview token user clicks button Confirm @@ -161,21 +162,21 @@ User revokes created API token user waits until modal is not visible Revoke preview token %{WAIT_LONG} user waits until page contains Generate API data set preview token -User again clicks on 'Generate Token' - user clicks button Generate token +User again clicks on 'Generate preview token' + user clicks button Generate preview token -User creates API token through 'Generate API token' modal window - ${modal}= user waits until modal is visible Generate API token - user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] API Token +User creates another preview token through 'Generate preview token' modal window + ${modal}= user waits until modal is visible Generate preview token + user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] ${PREVIEW_TOKEN_NAME} user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] user clicks button Continue user waits until page finishes loading - user waits until modal is not visible Generate API token %{WAIT_LONG} + user waits until modal is not visible Generate preview token %{WAIT_LONG} user waits until page contains API data set preview token user waits until h2 is visible ${SUBJECT_NAME_1} -User cancels to revoke created API token +User cancels revoking preview token user clicks button Revoke preview token ${modal}= user waits until modal is visible Revoke preview token user clicks button Cancel @@ -184,18 +185,18 @@ User cancels to revoke created API token user waits until page contains API data set preview token user waits until h2 is visible ${SUBJECT_NAME_1} -User cancels to create API token +User cancels creating preview token user clicks link Back to API data set details user clicks link containing text Preview API data set - user clicks button Generate token + user clicks button Generate preview token - ${modal}= user waits until modal is visible Generate API token - user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] API Token + ${modal}= user waits until modal is visible Generate preview token + user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] ${PREVIEW_TOKEN_NAME} user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] user clicks button Cancel user waits until page finishes loading - user waits until modal is not visible Generate API token %{WAIT_LONG} + user waits until modal is not visible Generate preview token %{WAIT_LONG} user waits until page contains Generate API data set preview token user waits until h2 is visible ${SUBJECT_NAME_1} @@ -221,19 +222,21 @@ Verify the 'Active tokens' and 'Expired tokens' on preview token log page user waits until page finishes loading user checks table cell contains 1 4 Expired -User verifies the relevant fields on the 'View Log Details' page for the Active API token. +User verifies the relevant fields on the active preview token page user clicks link Generate preview token - user clicks button Generate token + user clicks button Generate preview token - ${modal}= user waits until modal is visible Generate API token - user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] API Token + ${modal}= user waits until modal is visible Generate preview token + user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] ${PREVIEW_TOKEN_NAME} user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] user clicks button Continue user waits until page finishes loading - user waits until modal is not visible Generate API token %{WAIT_LONG} + user waits until modal is not visible Generate preview token %{WAIT_LONG} user waits until page contains API data set preview token + user waits until h2 is visible ${SUBJECT_NAME_1} + user checks page contains Reference: ${PREVIEW_TOKEN_NAME} ${current_time_tomorrow}= get current datetime %Y-%m-%dT%H:%M:%S 1 Europe/London @@ -265,7 +268,7 @@ Verify newly published release is on Find Statistics page User navigates to data catalogue page user navigates to data catalogue page on public frontend -Search with 1st API dataset +Search with 1st API data set user clicks element id:searchForm-search user presses keys ${PUBLICATION_NAME} user clicks radio API data sets only @@ -275,13 +278,13 @@ Search with 1st API dataset user checks page contains link ${SUBJECT_NAME_1} user checks list item contains testid:data-set-file-list 1 ${SUBJECT_NAME_1} -User clicks on API dataset link +User clicks on API data set link user clicks link by index ${SUBJECT_NAME_1} user waits until page finishes loading user waits until h1 is visible ${SUBJECT_NAME_1} -User checks relevant headings exist on API dataset details page +User checks relevant headings exist on API data set details page user waits until h2 is visible Data set details user waits until h2 is visible Data set preview user waits until h2 is visible Variables in this data set @@ -290,16 +293,16 @@ User checks relevant headings exist on API dataset details page user waits until h2 is visible API data set version history User verifies the row headings and contents in 'Data set details' section - user checks row headings within the api dataset section Theme - user checks row headings within the api dataset section Publication - user checks row headings within the api dataset section Release - user checks row headings within the api dataset section Release type - user checks row headings within the api dataset section Geographic levels - user checks row headings within the api dataset section Indicators - user checks row headings within the api dataset section Filters - user checks row headings within the api dataset section Time period - - user checks row headings within the api dataset section Notifications + user checks row headings within the api data set section Theme + user checks row headings within the api data set section Publication + user checks row headings within the api data set section Release + user checks row headings within the api data set section Release type + user checks row headings within the api data set section Geographic levels + user checks row headings within the api data set section Indicators + user checks row headings within the api data set section Filters + user checks row headings within the api data set section Time period + + user checks row headings within the api data set section Notifications user checks contents inside the cell value Test theme css: #dataSetDetails [data-testid="Theme-value"] user checks contents inside the cell value ${PUBLICATION_NAME} css:#dataSetDetails [data-testid="Publication-value"] diff --git a/tests/robot-tests/tests/public_api/public_api_resolve_mapping_statuses.robot b/tests/robot-tests/tests/public_api/public_api_resolve_mapping_statuses.robot index 562f4b88124..cc8f3b336c7 100644 --- a/tests/robot-tests/tests/public_api/public_api_resolve_mapping_statuses.robot +++ b/tests/robot-tests/tests/public_api/public_api_resolve_mapping_statuses.robot @@ -14,7 +14,7 @@ Test Setup fail test fast if required *** Variables *** -${PUBLICATION_NAME}= UI tests - public api resolve mapping statuses %{RUN_IDENTIFIER} +${PUBLICATION_NAME}= UI tests - Public API - resolve mapping statuses %{RUN_IDENTIFIER} ${RELEASE_NAME}= Financial year 3000-01 ${SUBJECT_NAME_1}= UI test subject 1 ${SUBJECT_NAME_2}= UI test subject 2 @@ -148,7 +148,7 @@ Validate the row headings and its contents in the 'Regions' section user checks table column heading contains 1 3 Type user checks table column heading contains 1 4 Actions - + user checks table cell contains 1 1 Yorkshire and The Humber user checks table cell contains 1 2 Unmapped user checks table cell contains 1 3 N/A @@ -241,7 +241,7 @@ Confirm finalization of this API data set version User navigates to 'changelog and guidance notes' page and update relevant details in it user clicks link by index View changelog and guidance notes 1 user waits until page contains API data set changelog - + user enters text into element css:textarea[id="guidanceNotesForm-notes"] public guidance notes user clicks button Save public guidance notes diff --git a/tests/robot-tests/tests/public_api/public_api_restricted.robot b/tests/robot-tests/tests/public_api/public_api_restricted.robot index a421b13cefa..76375cf9a5b 100644 --- a/tests/robot-tests/tests/public_api/public_api_restricted.robot +++ b/tests/robot-tests/tests/public_api/public_api_restricted.robot @@ -14,7 +14,7 @@ Test Setup fail test fast if required *** Variables *** -${PUBLICATION_NAME}= UI tests - public api restricted %{RUN_IDENTIFIER} +${PUBLICATION_NAME}= UI tests - Public API - restricted %{RUN_IDENTIFIER} ${RELEASE_NAME}= Financial year 3000-01 ${SUBJECT_NAME_1}= UI test subject 1 ${SUBJECT_NAME_2}= UI test subject 2 @@ -98,11 +98,11 @@ Verify the contents inside the 'Draft API datasets' table user checks table column heading contains 1 2 Name xpath://table[@data-testid='draft-api-data-sets'] user checks table column heading contains 1 3 Status xpath://table[@data-testid='draft-api-data-sets'] user checks table column heading contains 1 4 Actions xpath://table[@data-testid='draft-api-data-sets'] - + user checks table cell contains 1 1 v1.0 xpath://table[@data-testid='draft-api-data-sets'] user checks table cell contains 1 3 Ready xpath://table[@data-testid='draft-api-data-sets'] - + user checks table cell contains 2 1 v1.0 xpath://table[@data-testid='draft-api-data-sets'] user checks table cell contains 2 3 Ready xpath://table[@data-testid='draft-api-data-sets'] @@ -119,7 +119,7 @@ User checks row data contents inside the 'Draft API datasets' summary table user checks contents inside the cell value National xpath://div[@data-testid="Geographic levels"]//dd[@data-testid="Geographic levels-value"] user checks contents inside the cell value 2012/13 xpath://div[@data-testid="Time periods"]//dd[@data-testid="Time periods-value"] - + user checks contents inside the cell value Lower quartile annualised earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[1] user checks contents inside the cell value Median annualised earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[2] user checks contents inside the cell value Number of learners with earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[3] @@ -131,7 +131,7 @@ User checks row data contents inside the 'Draft API datasets' summary table user checks contents inside the cell value Cheese xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[1] user checks contents inside the cell value Colour xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[2] user checks contents inside the cell value Ethnicity group xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[3] - + user clicks button Show 4 more filters xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"] user checks contents inside the cell value Gender xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[4] @@ -156,7 +156,7 @@ Navigate to admin and create an amendment user navigates to admin dashboard Bau1 user creates amendment for release ${PUBLICATION_NAME} ${RELEASE_NAME} -Upload third subject (which is invalid for Public API import) into the first amendment +Upload third subject (which is invalid for Public API import) into the first amendment user uploads subject and waits until complete ... ${SUBJECT_NAME_3} ... invalid-data-set.csv @@ -206,7 +206,7 @@ Verify the contents inside the 'Live API datasets' table after the invalid impor user checks table cell contains 1 1 v1.0 xpath://table[@data-testid='live-api-data-sets'] user checks table cell contains 1 2 ${SUBJECT_NAME_1} xpath://table[@data-testid='live-api-data-sets'] - + user checks table cell contains 2 1 v1.0 xpath://table[@data-testid='live-api-data-sets'] user checks table cell contains 2 2 ${SUBJECT_NAME_2} xpath://table[@data-testid='live-api-data-sets'] @@ -258,7 +258,7 @@ Create a different version of an API dataset (minor version) user scrolls to the top of the page user clicks link API data sets user waits until h2 is visible API data sets - + user waits until h3 is visible Current live API data sets user checks table column heading contains 1 1 Version xpath://table[@data-testid="live-api-data-sets"] From dfee24653168de3c4ea70b431ac95a8b33c9bcf6 Mon Sep 17 00:00:00 2001 From: Mark Youngman Date: Fri, 4 Oct 2024 10:54:41 +0100 Subject: [PATCH 22/80] EES-2468 Add ability to delete users --- .../UserManagementControllerTests.cs | 2 +- .../UserManagementServicePermissionTests.cs | 31 +- .../Services/UserManagementServiceTests.cs | 8 +- .../Database/UsersAndRolesDbContext.cs | 6 + ...SetNullToUserInviteCreatedById.Designer.cs | 604 ++++++++++++++++++ ...S2468_AddSetNullToUserInviteCreatedById.cs | 41 ++ .../UsersAndRolesDbContextModelSnapshot.cs | 3 +- .../Services/UserManagementService.cs | 25 +- .../src/pages/bau/BauUsersPage.module.scss | 9 + .../src/pages/bau/BauUsersPage.tsx | 27 +- .../pages/bau/__tests__/BauUsersPage.test.tsx | 62 ++ .../src/services/userService.ts | 8 + 12 files changed, 767 insertions(+), 59 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/UsersAndRolesMigrations/20240924102651_EES2468_AddSetNullToUserInviteCreatedById.Designer.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/UsersAndRolesMigrations/20240924102651_EES2468_AddSetNullToUserInviteCreatedById.cs create mode 100644 src/explore-education-statistics-admin/src/pages/bau/BauUsersPage.module.scss create mode 100644 src/explore-education-statistics-admin/src/pages/bau/__tests__/BauUsersPage.test.tsx diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/UserManagement/UserManagementControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/UserManagement/UserManagementControllerTests.cs index cc82e11c3ef..51b871909ec 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/UserManagement/UserManagementControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/UserManagement/UserManagementControllerTests.cs @@ -13,7 +13,7 @@ public class UserManagementControllerTests(TestApplicationFactory testApp) : Int public class DeleteUserTests(TestApplicationFactory testApp) : UserManagementControllerTests(testApp) { [Theory] - [InlineData("BAU User", true)] + [InlineData("BAU User", false)] [InlineData("Analyst", false)] [InlineData("Prerelease User", false)] public async Task PermissionCheck( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserManagementServicePermissionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserManagementServicePermissionTests.cs index ea2ea6e9c20..f94f8cd3265 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserManagementServicePermissionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserManagementServicePermissionTests.cs @@ -131,39 +131,15 @@ public class DeleteUserTests public async Task Success() { await PolicyCheckBuilder() + .SetupCheck(CanManageUsersOnSystem) .AssertSuccess(async userService => { var service = SetupUserManagementService( - userService: userService.Object, - enableDeletion: true); + userService: userService.Object); return await service.DeleteUser("ees-test.user@education.gov.uk"); } ); } - - [Fact] - public async Task Forbidden_EnvironmentConfiguration() - { - await PolicyCheckBuilder() - .AssertForbidden(async userService => - { - var service = SetupUserManagementService(userService: userService.Object); - return await service.DeleteUser("ees.test-user@education.gov.uk"); - } - ); - } - - [Fact] - public async Task Forbidden_InvalidEmailAddressFormat() - { - await PolicyCheckBuilder() - .AssertForbidden(async userService => - { - var service = SetupUserManagementService(userService: userService.Object); - return await service.DeleteUser("invalid-email-to-delete@education.gov.uk"); - } - ); - } } private static UserManagementService SetupUserManagementService( @@ -194,8 +170,7 @@ private static UserManagementService SetupUserManagementService( userInviteRepository ?? new UserInviteRepository(usersAndRolesDbContext), userReleaseInviteRepository ?? new UserReleaseInviteRepository(contentDbContext), userPublicationInviteRepository ?? new UserPublicationInviteRepository(contentDbContext), - userManager ?? MockUserManager().Object, - new AppOptions { EnableThemeDeletion = enableDeletion }.ToOptionsWrapper() + userManager ?? MockUserManager().Object ); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserManagementServiceTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserManagementServiceTests.cs index 8bb120e4dd6..501b30c2c60 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserManagementServiceTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Services/UserManagementServiceTests.cs @@ -5,7 +5,6 @@ using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Database; using GovUk.Education.ExploreEducationStatistics.Admin.Models; -using GovUk.Education.ExploreEducationStatistics.Admin.Options; using GovUk.Education.ExploreEducationStatistics.Admin.Services; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Admin.Validators; @@ -17,7 +16,6 @@ using GovUk.Education.ExploreEducationStatistics.Content.Model; using GovUk.Education.ExploreEducationStatistics.Content.Model.Database; using Microsoft.AspNetCore.Identity; -using Microsoft.Extensions.Configuration; using Moq; using static GovUk.Education.ExploreEducationStatistics.Admin.Tests.Services.DbUtils; using static GovUk.Education.ExploreEducationStatistics.Admin.Tests.Utils.AdminMockUtils; @@ -1030,8 +1028,7 @@ private static UserManagementService SetupUserManagementService( IUserInviteRepository? userInviteRepository = null, IUserReleaseInviteRepository? userReleaseInviteRepository = null, IUserPublicationInviteRepository? userPublicationInviteRepository = null, - UserManager? userManager = null, - IConfiguration? configuration = null) + UserManager? userManager = null) { contentDbContext ??= InMemoryApplicationDbContext(); usersAndRolesDbContext ??= InMemoryUserAndRolesDbContext(); @@ -1047,8 +1044,7 @@ private static UserManagementService SetupUserManagementService( userInviteRepository ?? new UserInviteRepository(usersAndRolesDbContext), userReleaseInviteRepository ?? new UserReleaseInviteRepository(contentDbContext), userPublicationInviteRepository ?? new UserPublicationInviteRepository(contentDbContext), - userManager ?? MockUserManager().Object, - new AppOptions { EnableThemeDeletion = true }.ToOptionsWrapper() + userManager ?? MockUserManager().Object ); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Database/UsersAndRolesDbContext.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Database/UsersAndRolesDbContext.cs index 12dc1d29ab5..4d312a26982 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Database/UsersAndRolesDbContext.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Database/UsersAndRolesDbContext.cs @@ -47,6 +47,12 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) base.OnModelCreating(modelBuilder); modelBuilder.Entity() + .HasOne(c => c.CreatedBy) + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.SetNull); + + modelBuilder.Entity() .Property(invite => invite.Created) .HasConversion( v => v, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/UsersAndRolesMigrations/20240924102651_EES2468_AddSetNullToUserInviteCreatedById.Designer.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/UsersAndRolesMigrations/20240924102651_EES2468_AddSetNullToUserInviteCreatedById.Designer.cs new file mode 100644 index 00000000000..7153a063f4a --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/UsersAndRolesMigrations/20240924102651_EES2468_AddSetNullToUserInviteCreatedById.Designer.cs @@ -0,0 +1,604 @@ +// +using System; +using GovUk.Education.ExploreEducationStatistics.Admin.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Migrations.UsersAndRolesMigrations +{ + [DbContext(typeof(UsersAndRolesDbContext))] + [Migration("20240924102651_EES2468_AddSetNullToUserInviteCreatedById")] + partial class EES2468_AddSetNullToUserInviteCreatedById + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Admin.Models.ApplicationUser", b => + { + b.Property("Id") + .HasColumnType("nvarchar(450)"); + + b.Property("AccessFailedCount") + .HasColumnType("int"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken() + .HasColumnType("nvarchar(max)"); + + b.Property("Email") + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("EmailConfirmed") + .HasColumnType("bit"); + + b.Property("FirstName") + .HasColumnType("nvarchar(max)"); + + b.Property("LastName") + .HasColumnType("nvarchar(max)"); + + b.Property("LockoutEnabled") + .HasColumnType("bit"); + + b.Property("LockoutEnd") + .HasColumnType("datetimeoffset"); + + b.Property("NormalizedEmail") + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("NormalizedUserName") + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("PasswordHash") + .HasColumnType("nvarchar(max)"); + + b.Property("PhoneNumber") + .HasColumnType("nvarchar(max)"); + + b.Property("PhoneNumberConfirmed") + .HasColumnType("bit"); + + b.Property("SecurityStamp") + .HasColumnType("nvarchar(max)"); + + b.Property("TwoFactorEnabled") + .HasColumnType("bit"); + + b.Property("UserName") + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedEmail") + .HasDatabaseName("EmailIndex"); + + b.HasIndex("NormalizedUserName") + .IsUnique() + .HasDatabaseName("UserNameIndex") + .HasFilter("[NormalizedUserName] IS NOT NULL"); + + b.ToTable("AspNetUsers", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Admin.Models.UserInvite", b => + { + b.Property("Email") + .HasColumnType("nvarchar(450)"); + + b.Property("Accepted") + .HasColumnType("bit"); + + b.Property("Created") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("nvarchar(450)"); + + b.Property("RoleId") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.HasKey("Email"); + + b.HasIndex("CreatedById"); + + b.HasIndex("RoleId"); + + b.ToTable("UserInvites"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRole", b => + { + b.Property("Id") + .HasColumnType("nvarchar(450)"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken() + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("NormalizedName") + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedName") + .IsUnique() + .HasDatabaseName("RoleNameIndex") + .HasFilter("[NormalizedName] IS NOT NULL"); + + b.ToTable("AspNetRoles", (string)null); + + b.HasData( + new + { + Id = "cf67b697-bddd-41bd-86e0-11b7e11d99b3", + ConcurrencyStamp = "85d6c75e-a6c8-4c7e-b4d0-8ee70a4879d3", + Name = "BAU User", + NormalizedName = "BAU USER" + }, + new + { + Id = "f9ddb43e-aa9e-41ed-837d-3062e130c425", + ConcurrencyStamp = "85d6c75e-a6c8-4c7e-b4d0-8ee70a4879d3", + Name = "Analyst", + NormalizedName = "ANALYST" + }, + new + { + Id = "17e634f4-7a2b-4a23-8636-b079877b4232", + ConcurrencyStamp = "85d6c75e-a6c8-4c7e-b4d0-8ee70a4879d3", + Name = "Prerelease User", + NormalizedName = "PRERELEASE USER" + }); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ClaimType") + .HasColumnType("nvarchar(max)"); + + b.Property("ClaimValue") + .HasColumnType("nvarchar(max)"); + + b.Property("RoleId") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetRoleClaims", (string)null); + + b.HasData( + new + { + Id = -2, + ClaimType = "ApplicationAccessGranted", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -3, + ClaimType = "AccessAllReleases", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -5, + ClaimType = "MarkAllReleasesAsDraft", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -6, + ClaimType = "SubmitAllReleasesToHigherReview", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -7, + ClaimType = "ApproveAllReleases", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -8, + ClaimType = "UpdateAllReleases", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -9, + ClaimType = "CreateAnyPublication", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -10, + ClaimType = "CreateAnyRelease", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -11, + ClaimType = "ManageAnyUser", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -13, + ClaimType = "ApplicationAccessGranted", + ClaimValue = "", + RoleId = "f9ddb43e-aa9e-41ed-837d-3062e130c425" + }, + new + { + Id = -14, + ClaimType = "AnalystPagesAccessGranted", + ClaimValue = "", + RoleId = "f9ddb43e-aa9e-41ed-837d-3062e130c425" + }, + new + { + Id = -15, + ClaimType = "ApplicationAccessGranted", + ClaimValue = "", + RoleId = "17e634f4-7a2b-4a23-8636-b079877b4232" + }, + new + { + Id = -16, + ClaimType = "PrereleasePagesAccessGranted", + ClaimValue = "", + RoleId = "17e634f4-7a2b-4a23-8636-b079877b4232" + }, + new + { + Id = -17, + ClaimType = "PrereleasePagesAccessGranted", + ClaimValue = "", + RoleId = "f9ddb43e-aa9e-41ed-837d-3062e130c425" + }, + new + { + Id = -18, + ClaimType = "AnalystPagesAccessGranted", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -19, + ClaimType = "PrereleasePagesAccessGranted", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -20, + ClaimType = "CanViewPrereleaseContacts", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -21, + ClaimType = "CanViewPrereleaseContacts", + ClaimValue = "", + RoleId = "f9ddb43e-aa9e-41ed-837d-3062e130c425" + }, + new + { + Id = -22, + ClaimType = "CreateAnyMethodology", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -23, + ClaimType = "UpdateAllMethodologies", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -24, + ClaimType = "AccessAllMethodologies", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -29, + ClaimType = "ApproveAllMethodologies", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -30, + ClaimType = "PublishAllReleases", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -31, + ClaimType = "MakeAmendmentsOfAllReleases", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -32, + ClaimType = "DeleteAllReleaseAmendments", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -33, + ClaimType = "ManageAllTaxonomy", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -34, + ClaimType = "UpdateAllPublications", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -35, + ClaimType = "MarkAllMethodologiesDraft", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -36, + ClaimType = "AccessAllImports", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -37, + ClaimType = "CancelAllFileImports", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -38, + ClaimType = "MakeAmendmentsOfAllMethodologies", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -39, + ClaimType = "DeleteAllMethodologies", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -40, + ClaimType = "AdoptAnyMethodology", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -41, + ClaimType = "AccessAllPublications", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }, + new + { + Id = -42, + ClaimType = "SubmitAllMethodologiesToHigherReview", + ClaimValue = "", + RoleId = "cf67b697-bddd-41bd-86e0-11b7e11d99b3" + }); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ClaimType") + .HasColumnType("nvarchar(max)"); + + b.Property("ClaimValue") + .HasColumnType("nvarchar(max)"); + + b.Property("UserId") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserClaims", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.Property("LoginProvider") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("ProviderKey") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("ProviderDisplayName") + .HasColumnType("nvarchar(max)"); + + b.Property("UserId") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.HasKey("LoginProvider", "ProviderKey"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserLogins", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.Property("UserId") + .HasColumnType("nvarchar(450)"); + + b.Property("RoleId") + .HasColumnType("nvarchar(450)"); + + b.HasKey("UserId", "RoleId"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetUserRoles", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.Property("UserId") + .HasColumnType("nvarchar(450)"); + + b.Property("LoginProvider") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Name") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Value") + .HasColumnType("nvarchar(max)"); + + b.HasKey("UserId", "LoginProvider", "Name"); + + b.ToTable("AspNetUserTokens", (string)null); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Admin.Models.UserInvite", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Admin.Models.ApplicationUser", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", "Role") + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("Role"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Admin.Models.ApplicationUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Admin.Models.ApplicationUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Admin.Models.ApplicationUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Admin.Models.ApplicationUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/UsersAndRolesMigrations/20240924102651_EES2468_AddSetNullToUserInviteCreatedById.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/UsersAndRolesMigrations/20240924102651_EES2468_AddSetNullToUserInviteCreatedById.cs new file mode 100644 index 00000000000..dac4fca1ade --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/UsersAndRolesMigrations/20240924102651_EES2468_AddSetNullToUserInviteCreatedById.cs @@ -0,0 +1,41 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Admin.Migrations.UsersAndRolesMigrations +{ + /// + public partial class EES2468_AddSetNullToUserInviteCreatedById : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "FK_UserInvites_AspNetUsers_CreatedById", + table: "UserInvites"); + + migrationBuilder.AddForeignKey( + name: "FK_UserInvites_AspNetUsers_CreatedById", + table: "UserInvites", + column: "CreatedById", + principalTable: "AspNetUsers", + principalColumn: "Id", + onDelete: ReferentialAction.SetNull); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "FK_UserInvites_AspNetUsers_CreatedById", + table: "UserInvites"); + + migrationBuilder.AddForeignKey( + name: "FK_UserInvites_AspNetUsers_CreatedById", + table: "UserInvites", + column: "CreatedById", + principalTable: "AspNetUsers", + principalColumn: "Id"); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/UsersAndRolesMigrations/UsersAndRolesDbContextModelSnapshot.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/UsersAndRolesMigrations/UsersAndRolesDbContextModelSnapshot.cs index 003cc448624..bb0b62257f8 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/UsersAndRolesMigrations/UsersAndRolesDbContextModelSnapshot.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Migrations/UsersAndRolesMigrations/UsersAndRolesDbContextModelSnapshot.cs @@ -531,7 +531,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) { b.HasOne("GovUk.Education.ExploreEducationStatistics.Admin.Models.ApplicationUser", "CreatedBy") .WithMany() - .HasForeignKey("CreatedById"); + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.SetNull); b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", "Role") .WithMany() diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/UserManagementService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/UserManagementService.cs index 8a53b640bf9..a66d17bf3a6 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/UserManagementService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/UserManagementService.cs @@ -18,7 +18,6 @@ using Microsoft.AspNetCore.Identity; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Options; using static GovUk.Education.ExploreEducationStatistics.Admin.Models.GlobalRoles; using static GovUk.Education.ExploreEducationStatistics.Admin.Validators.ValidationUtils; using static GovUk.Education.ExploreEducationStatistics.Admin.Validators.ValidationErrorMessages; @@ -40,7 +39,6 @@ public class UserManagementService : IUserManagementService private readonly IUserReleaseInviteRepository _userReleaseInviteRepository; private readonly IUserPublicationInviteRepository _userPublicationInviteRepository; private readonly UserManager _userManager; - private readonly bool _userDeletionAllowed; public UserManagementService( UsersAndRolesDbContext usersAndRolesDbContext, @@ -53,8 +51,7 @@ public UserManagementService( IUserInviteRepository userInviteRepository, IUserReleaseInviteRepository userReleaseInviteRepository, IUserPublicationInviteRepository userPublicationInviteRepository, - UserManager userManager, - IOptions appOptions) + UserManager userManager) { _usersAndRolesDbContext = usersAndRolesDbContext; _contentDbContext = contentDbContext; @@ -67,7 +64,6 @@ public UserManagementService( _userReleaseInviteRepository = userReleaseInviteRepository; _userPublicationInviteRepository = userPublicationInviteRepository; _userManager = userManager; - _userDeletionAllowed = appOptions.Value.EnableThemeDeletion; } public async Task>> ListAllUsers() @@ -420,7 +416,8 @@ public async Task> UpdateUser(string userId, string r public async Task> DeleteUser(string email) { - return await CheckCanDeleteUser(email) + return await _userService + .CheckCanManageAllUsers() .OnSuccessDo(async () => { // Delete the Identity Framework user, if found. @@ -468,22 +465,6 @@ public async Task> DeleteUser(string email) }); } - private async Task> CheckCanDeleteUser(string email) - { - if (!_userDeletionAllowed) - { - return new ForbidResult(); - } - - // We currently only allow test users to be deleted. - if (!email.ToLower().StartsWith("ees-test.")) - { - return new ForbidResult(); - } - - return Unit.Instance; - } - private async Task> ValidateUserDoesNotExist(string email) { if (await _usersAndRolesDbContext.Users diff --git a/src/explore-education-statistics-admin/src/pages/bau/BauUsersPage.module.scss b/src/explore-education-statistics-admin/src/pages/bau/BauUsersPage.module.scss new file mode 100644 index 00000000000..d4c440652f4 --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/bau/BauUsersPage.module.scss @@ -0,0 +1,9 @@ +@import '~govuk-frontend/dist/govuk/base'; + +.manageUserLink { + margin-right: govuk-spacing(2); +} + +.deleteUserButton { + color: govuk-colour('red'); +} diff --git a/src/explore-education-statistics-admin/src/pages/bau/BauUsersPage.tsx b/src/explore-education-statistics-admin/src/pages/bau/BauUsersPage.tsx index edef343bbd6..6a0551f41ef 100644 --- a/src/explore-education-statistics-admin/src/pages/bau/BauUsersPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/bau/BauUsersPage.tsx @@ -1,13 +1,27 @@ import Link from '@admin/components/Link'; import Page from '@admin/components/Page'; import userService from '@admin/services/userService'; +import ButtonText from '@common/components/ButtonText'; import LoadingSpinner from '@common/components/LoadingSpinner'; import useAsyncRetry from '@common/hooks/useAsyncRetry'; +import logger from '@common/services/logger'; import React from 'react'; +import styles from './BauUsersPage.module.scss'; const BauUsersPage = () => { const { value, isLoading } = useAsyncRetry(() => userService.getUsers()); + const handleDeleteUser = async (userEmail: string) => { + await userService + .deleteUser(userEmail) + .then(() => { + window.location.reload(); + }) + .catch(error => { + logger.info(`Error encountered when deleting the user - ${error}`); + }); + }; + return ( { {user.email} {user.role ?? 'No role'} - Manage + + Manage + + handleDeleteUser(user.email)} + className={styles.deleteUserButton} + > + Delete + ))} diff --git a/src/explore-education-statistics-admin/src/pages/bau/__tests__/BauUsersPage.test.tsx b/src/explore-education-statistics-admin/src/pages/bau/__tests__/BauUsersPage.test.tsx new file mode 100644 index 00000000000..4249d7ace84 --- /dev/null +++ b/src/explore-education-statistics-admin/src/pages/bau/__tests__/BauUsersPage.test.tsx @@ -0,0 +1,62 @@ +import _userService, { + RemoveUser, + UserStatus, +} from '@admin/services/userService'; +import { MemoryRouter } from 'react-router'; +import { TestConfigContextProvider } from '@admin/contexts/ConfigContext'; +import { render, screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import BauUsersPage from '../BauUsersPage'; + +jest.mock('@admin/services/userService'); + +const userService = _userService as jest.Mocked; + +const user: UserStatus[] = [ + { + id: '1', + name: 'TestUser1', + email: 'test@hotmail.com', + role: 'test', + }, +]; + +const removedUser: RemoveUser = { + userId: '1', +}; + +describe('BauUsersPage', () => { + test('renders delete action when user a user is present', async () => { + userService.getUsers.mockResolvedValue(user); + + renderPage(); + + await waitFor(() => { + expect(screen.getByText('Delete')).toBeInTheDocument(); + }); + }); + + test('calls user service when delete user button is clicked', async () => { + userService.getUsers.mockResolvedValue(user); + userService.deleteUser.mockResolvedValue(Promise.resolve(removedUser)); + + renderPage(); + + await waitFor(() => { + expect(screen.getByText('Delete')).toBeInTheDocument(); + }); + await userEvent.click(screen.getByRole('button', { name: 'Delete' })); + + expect(userService.deleteUser).toHaveBeenCalled(); + }); + + function renderPage() { + return render( + + + + + , + ); + } +}); diff --git a/src/explore-education-statistics-admin/src/services/userService.ts b/src/explore-education-statistics-admin/src/services/userService.ts index 0d6015a3115..15c11cabb8f 100644 --- a/src/explore-education-statistics-admin/src/services/userService.ts +++ b/src/explore-education-statistics-admin/src/services/userService.ts @@ -72,11 +72,16 @@ export interface ResourceRoles { Release?: string[]; } +export interface RemoveUser { + userId: string; +} + export interface UsersService { getRoles(): Promise; getReleases(): Promise; getResourceRoles(): Promise; getUser(userId: string): Promise; + deleteUser(email: string): Promise; addUserReleaseRole: ( userId: string, userReleaseRole: UserReleaseRoleSubmission, @@ -129,6 +134,9 @@ const userService: UsersService = { updateUser(userId: string, update: UserUpdate): Promise { return client.put(`/user-management/users/${userId}`, update); }, + deleteUser(email: string): Promise { + return client.delete(`/user-management/user/${email}`); + }, addUserReleaseRole( userId: string, From 35843d2a769ed27994b100ade7682012ea2cf9f2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 3 Oct 2024 12:00:49 +0000 Subject: [PATCH 23/80] Update peter-evans/create-pull-request action to v7 --- .github/workflows/validate-snapshots.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-snapshots.yml b/.github/workflows/validate-snapshots.yml index fb5297c5fd3..fbc095016ff 100644 --- a/.github/workflows/validate-snapshots.yml +++ b/.github/workflows/validate-snapshots.yml @@ -57,7 +57,7 @@ jobs: - name: Create pull request id: cpr - uses: peter-evans/create-pull-request@v6 + uses: peter-evans/create-pull-request@v7 with: token: ${{ env.GH_TOKEN }} commit-message: 'chore(tests): update test snapshots' From 0de719f82d6661f072948024e30a9bb989b74f38 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 3 Oct 2024 12:00:38 +0000 Subject: [PATCH 24/80] Update slackapi/slack-github-action action to v1.27.0 --- .github/workflows/validate-snapshots.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-snapshots.yml b/.github/workflows/validate-snapshots.yml index fbc095016ff..3ce9add9d38 100644 --- a/.github/workflows/validate-snapshots.yml +++ b/.github/workflows/validate-snapshots.yml @@ -84,7 +84,7 @@ jobs: token: ${{ env.GH_TOKEN }} - name: Slack Notification - uses: slackapi/slack-github-action@v1.26.0 + uses: slackapi/slack-github-action@v1.27.0 with: payload: | { From bce50e0dd315d466955e12d8d1444ca38c16907d Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Fri, 4 Oct 2024 14:54:41 +0100 Subject: [PATCH 25/80] EES-5555 Fix pre-commit hooks not working This fix simply upgrades husky and lint-staged to the latest. --- .husky/pre-commit | 7 +- package.json | 6 +- pnpm-lock.yaml | 282 ++++++++++++++++++++++++++++------------------ 3 files changed, 175 insertions(+), 120 deletions(-) diff --git a/.husky/pre-commit b/.husky/pre-commit index 73b31bb0ffc..c27d8893a99 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,6 +1 @@ -#!/bin/sh -. "$(dirname "$0")/_/husky.sh" - -if [ -d "./node_modules/.bin/lint-staged" ]; then - ./node_modules/.bin/lint-staged -fi \ No newline at end of file +lint-staged diff --git a/package.json b/package.json index bc421b8fd11..a33cbc3fbfb 100644 --- a/package.json +++ b/package.json @@ -32,8 +32,8 @@ "eslint-plugin-react": "^7.32.2", "eslint-plugin-react-hooks": "^4.6.0", "execa": "^8.0.1", - "husky": "^8.0.3", - "lint-staged": "^13.2.3", + "husky": "^9.1.6", + "lint-staged": "^15.2.10", "lodash": "^4.17.21", "prettier": "^3.0.1", "signal-exit": "^4.1.0", @@ -46,7 +46,7 @@ "typescript": "^5.5.4" }, "scripts": { - "prepare": "husky install", + "prepare": "husky", "preinstall": "pnpm check:node", "clean": "pnpm -r --parallel exec rm -rf node_modules && rm -rf node_modules && rm -rf src/explore-education-statistics-frontend/.next", "check:node": "check-node-version --package", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fee0f5a75a8..041617ce83d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -81,11 +81,11 @@ importers: specifier: ^8.0.1 version: 8.0.1 husky: - specifier: ^8.0.3 - version: 8.0.3 + specifier: ^9.1.6 + version: 9.1.6 lint-staged: - specifier: ^13.2.3 - version: 13.2.3 + specifier: ^15.2.10 + version: 15.2.10 lodash: specifier: ^4.17.21 version: 4.17.21 @@ -7101,6 +7101,13 @@ packages: type-fest: 3.13.0 dev: true + /ansi-escapes@7.0.0: + resolution: {integrity: sha512-GdYO7a61mR0fOlAsvC9/rIHf7L96sBc6dEWzeOu+KAea5bZyQRPIpojrVoI4AXGJS/ycu/fBTdLrUkA4ODrvjw==} + engines: {node: '>=18'} + dependencies: + environment: 1.1.0 + dev: true + /ansi-html-community@0.0.8: resolution: {integrity: sha512-1APHAyr3+PCamwNw3bXCPp4HFLONZt/yIH0sZp0/469KWNTEy+qN5jQ3GVX6DMZ1UXAi34yVwtTeaG/HpBuuzw==} engines: {'0': node >= 0.8.0} @@ -7795,6 +7802,13 @@ packages: dependencies: fill-range: 7.0.1 + /braces@3.0.3: + resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} + engines: {node: '>=8'} + dependencies: + fill-range: 7.1.1 + dev: true + /browserslist@4.21.5: resolution: {integrity: sha512-tUkiguQGW7S3IhB7N+c2MV/HZPSCPAAiYBZXLsBhFB/PCy6ZKKsZrmBayHV9fdGV/ARIfJ14NkxKzRDjvp7L6w==} engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} @@ -8010,11 +8024,6 @@ packages: ansi-styles: 4.3.0 supports-color: 7.2.0 - /chalk@5.2.0: - resolution: {integrity: sha512-ree3Gqw/nazQAPuJJEy+avdl7QfZMcUvmHIKgEZkGL+xOBzRvup5Hxo6LHuMceSxOabuJLJm5Yp/92R9eMmMvA==} - engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} - dev: true - /chalk@5.3.0: resolution: {integrity: sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==} engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} @@ -8138,27 +8147,19 @@ packages: engines: {node: '>=6'} dev: true - /cli-cursor@3.1.0: - resolution: {integrity: sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==} - engines: {node: '>=8'} - dependencies: - restore-cursor: 3.1.0 - dev: true - - /cli-truncate@2.1.0: - resolution: {integrity: sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg==} - engines: {node: '>=8'} + /cli-cursor@5.0.0: + resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} + engines: {node: '>=18'} dependencies: - slice-ansi: 3.0.0 - string-width: 4.2.3 + restore-cursor: 5.1.0 dev: true - /cli-truncate@3.1.0: - resolution: {integrity: sha512-wfOBkjXteqSnI59oPcJkcPl/ZmwvMMOj340qUIY1SKZCv0B9Cf4D4fAucRkIKQmsIuYK3x1rrgU7MeGRruiuiA==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + /cli-truncate@4.0.0: + resolution: {integrity: sha512-nPdaFdQ0h/GEigbPClz11D0v/ZJEwxmeVZGeMo3Z5StPtUTkA9o1lD6QwoirYiSDzbcwn2XcjwmCp68W1IS4TA==} + engines: {node: '>=18'} dependencies: slice-ansi: 5.0.0 - string-width: 5.1.2 + string-width: 7.2.0 dev: true /client-only@0.0.1: @@ -8267,6 +8268,10 @@ packages: /colorette@2.0.19: resolution: {integrity: sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==} + /colorette@2.0.20: + resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} + dev: true + /combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} @@ -8290,6 +8295,11 @@ packages: engines: {node: '>=16'} dev: true + /commander@12.1.0: + resolution: {integrity: sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==} + engines: {node: '>=18'} + dev: true + /commander@2.20.3: resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==} @@ -8986,6 +8996,18 @@ packages: dependencies: ms: 2.1.2 + /debug@4.3.7: + resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.1.3 + dev: true + /decamelize-keys@1.1.1: resolution: {integrity: sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==} engines: {node: '>=0.10.0'} @@ -9382,10 +9404,6 @@ packages: resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==} dev: false - /eastasianwidth@0.2.0: - resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} - dev: true - /ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} @@ -9414,6 +9432,10 @@ packages: engines: {node: '>=12'} dev: true + /emoji-regex@10.4.0: + resolution: {integrity: sha512-EC+0oUMY1Rqm4O6LLrgjtYDvcVYTy7chDnM4Q7030tP4Kwj3u/pR6gP9ygnp2CJMK5Gq+9Q2oqmrFJAz01DXjw==} + dev: true + /emoji-regex@8.0.0: resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} dev: true @@ -9465,6 +9487,11 @@ packages: engines: {node: '>=4'} hasBin: true + /environment@1.1.0: + resolution: {integrity: sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==} + engines: {node: '>=18'} + dev: true + /error-ex@1.3.2: resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} dependencies: @@ -10219,6 +10246,10 @@ packages: /eventemitter3@4.0.7: resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} + /eventemitter3@5.0.1: + resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} + dev: true + /events@3.3.0: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} @@ -10514,6 +10545,13 @@ packages: dependencies: to-regex-range: 5.0.1 + /fill-range@7.1.1: + resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} + engines: {node: '>=8'} + dependencies: + to-regex-range: 5.0.1 + dev: true + /finalhandler@1.2.0: resolution: {integrity: sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==} engines: {node: '>= 0.8'} @@ -10771,6 +10809,11 @@ packages: engines: {node: 6.* || 8.* || >= 10.*} dev: true + /get-east-asian-width@1.2.0: + resolution: {integrity: sha512-2nk+7SIVb14QrgXFHcm84tD4bKQz0RxPuMT8Ag5KPOq7J5fEmAg0UbXdTOSHqNuHSU28k55qnceesxXRZGzKWA==} + engines: {node: '>=18'} + dev: true + /get-intrinsic@1.2.0: resolution: {integrity: sha512-L049y6nFOuom5wGyRc3/gdTLO94dySVKRACj1RmJZBQXlbTMhtNIgkWkUHq+jYmZvKf14EW1EoJnnjbmoHij0Q==} dependencies: @@ -11345,9 +11388,9 @@ packages: engines: {node: '>=16.17.0'} dev: true - /husky@8.0.3: - resolution: {integrity: sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg==} - engines: {node: '>=14'} + /husky@9.1.6: + resolution: {integrity: sha512-sqbjZKK7kf44hfdE94EoX8MZNk0n7HeW37O4YrVGCF4wzgQjp+akPAkfUK5LZ6KuR/6sqeAVuXHji+RzQgOn5A==} + engines: {node: '>=18'} hasBin: true dev: true @@ -11738,6 +11781,13 @@ packages: engines: {node: '>=12'} dev: true + /is-fullwidth-code-point@5.0.0: + resolution: {integrity: sha512-OVa3u9kkBbw7b8Xw5F9P+D/T9X+Z4+JruYVNapTjPYZYUznQ5YfWeFkOj606XYYW8yugTfC8Pj0hYqvi4ryAhA==} + engines: {node: '>=18'} + dependencies: + get-east-asian-width: 1.2.0 + dev: true + /is-generator-fn@2.1.0: resolution: {integrity: sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==} engines: {node: '>=6'} @@ -12747,50 +12797,45 @@ packages: /lilconfig@2.1.0: resolution: {integrity: sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==} engines: {node: '>=10'} + dev: false + + /lilconfig@3.1.2: + resolution: {integrity: sha512-eop+wDAvpItUys0FWkHIKeC9ybYrTGbU41U5K7+bttZZeohvnY7M9dZ5kB21GNWiFT2q1OoPTvncPCgSOVO5ow==} + engines: {node: '>=14'} + dev: true /lines-and-columns@1.2.4: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} - /lint-staged@13.2.3: - resolution: {integrity: sha512-zVVEXLuQIhr1Y7R7YAWx4TZLdvuzk7DnmrsTNL0fax6Z3jrpFcas+vKbzxhhvp6TA55m1SQuWkpzI1qbfDZbAg==} - engines: {node: ^14.13.1 || >=16.0.0} + /lint-staged@15.2.10: + resolution: {integrity: sha512-5dY5t743e1byO19P9I4b3x8HJwalIznL5E1FWYnU6OWw33KxNBSLAc6Cy7F2PsFEO8FKnLwjwm5hx7aMF0jzZg==} + engines: {node: '>=18.12.0'} hasBin: true dependencies: - chalk: 5.2.0 - cli-truncate: 3.1.0 - commander: 10.0.1 - debug: 4.3.4 - execa: 7.1.1 - lilconfig: 2.1.0 - listr2: 5.0.8 - micromatch: 4.0.5 - normalize-path: 3.0.0 - object-inspect: 1.12.3 + chalk: 5.3.0 + commander: 12.1.0 + debug: 4.3.7 + execa: 8.0.1 + lilconfig: 3.1.2 + listr2: 8.2.5 + micromatch: 4.0.8 pidtree: 0.6.0 - string-argv: 0.3.1 - yaml: 2.3.1 + string-argv: 0.3.2 + yaml: 2.5.1 transitivePeerDependencies: - - enquirer - supports-color dev: true - /listr2@5.0.8: - resolution: {integrity: sha512-mC73LitKHj9w6v30nLNGPetZIlfpUniNSsxxrbaPcWOjDb92SHPzJPi/t+v1YC/lxKz/AJ9egOjww0qUuFxBpA==} - engines: {node: ^14.13.1 || >=16.0.0} - peerDependencies: - enquirer: '>= 2.3.0 < 3' - peerDependenciesMeta: - enquirer: - optional: true + /listr2@8.2.5: + resolution: {integrity: sha512-iyAZCeyD+c1gPyE9qpFu8af0Y+MRtmKOncdGoA2S5EY8iFq99dmmvkNnHiWo+pj0s7yH7l3KPIgee77tKpXPWQ==} + engines: {node: '>=18.0.0'} dependencies: - cli-truncate: 2.1.0 - colorette: 2.0.19 - log-update: 4.0.0 - p-map: 4.0.0 - rfdc: 1.3.0 - rxjs: 7.8.0 - through: 2.3.8 - wrap-ansi: 7.0.0 + cli-truncate: 4.0.0 + colorette: 2.0.20 + eventemitter3: 5.0.1 + log-update: 6.1.0 + rfdc: 1.4.1 + wrap-ansi: 9.0.0 dev: true /loader-runner@4.3.0: @@ -12872,14 +12917,15 @@ packages: /lodash@4.17.21: resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} - /log-update@4.0.0: - resolution: {integrity: sha512-9fkkDevMefjg0mmzWFBW8YkFP91OrizzkW3diF7CpG+S2EYdy4+TVfGwz1zeF8x7hCx1ovSPTOE9Ngib74qqUg==} - engines: {node: '>=10'} + /log-update@6.1.0: + resolution: {integrity: sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==} + engines: {node: '>=18'} dependencies: - ansi-escapes: 4.3.2 - cli-cursor: 3.1.0 - slice-ansi: 4.0.0 - wrap-ansi: 6.2.0 + ansi-escapes: 7.0.0 + cli-cursor: 5.0.0 + slice-ansi: 7.1.0 + strip-ansi: 7.1.0 + wrap-ansi: 9.0.0 dev: true /loose-envify@1.4.0: @@ -13318,6 +13364,14 @@ packages: braces: 3.0.2 picomatch: 2.3.1 + /micromatch@4.0.8: + resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} + engines: {node: '>=8.6'} + dependencies: + braces: 3.0.3 + picomatch: 2.3.1 + dev: true + /mime-db@1.52.0: resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} engines: {node: '>= 0.6'} @@ -13342,6 +13396,11 @@ packages: engines: {node: '>=12'} dev: true + /mimic-function@5.0.1: + resolution: {integrity: sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==} + engines: {node: '>=18'} + dev: true + /min-document@2.19.0: resolution: {integrity: sha512-9Wy1B3m3f66bPPmU5hdA4DR4PB2OfDU/+GS3yAB7IQozE3tqXaVv2zOjgla7MEGSRv95+ILmOuvhLkOK6wJtCQ==} dependencies: @@ -13872,6 +13931,13 @@ packages: mimic-fn: 4.0.0 dev: true + /onetime@7.0.0: + resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==} + engines: {node: '>=18'} + dependencies: + mimic-function: 5.0.1 + dev: true + /open@8.4.1: resolution: {integrity: sha512-/4b7qZNhv6Uhd7jjnREh1NjnPxlTq+XNWPG88Ydkj5AILcA5m3ajvcg57pB24EQjKv0dK62XnDqk9c/hkIG5Kg==} engines: {node: '>=12'} @@ -16273,12 +16339,12 @@ packages: supports-preserve-symlinks-flag: 1.0.0 dev: true - /restore-cursor@3.1.0: - resolution: {integrity: sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==} - engines: {node: '>=8'} + /restore-cursor@5.1.0: + resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} + engines: {node: '>=18'} dependencies: - onetime: 5.1.2 - signal-exit: 3.0.7 + onetime: 7.0.0 + signal-exit: 4.1.0 dev: true /resumer@0.0.0: @@ -16300,8 +16366,8 @@ packages: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} - /rfdc@1.3.0: - resolution: {integrity: sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==} + /rfdc@1.4.1: + resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==} dev: true /rgb-regex@1.0.1: @@ -16351,12 +16417,6 @@ packages: dependencies: queue-microtask: 1.2.3 - /rxjs@7.8.0: - resolution: {integrity: sha512-F2+gxDshqmIub1KdvZkaEfGDwLNpPvk9Fs6LD/MyQxNgMds/WH9OdDDXOmxUZpME+iSK3rQCctkL0DYyytUqMg==} - dependencies: - tslib: 2.6.0 - dev: true - /sade@1.8.1: resolution: {integrity: sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==} engines: {node: '>=6'} @@ -16705,15 +16765,6 @@ packages: engines: {node: '>=14.16'} dev: true - /slice-ansi@3.0.0: - resolution: {integrity: sha512-pSyv7bSTC7ig9Dcgbw9AuRNUb5k5V6oDudjZoMBSr13qpLBG7tB+zgCkARjq7xIUgdz5P1Qe8u+rSGdouOOIyQ==} - engines: {node: '>=8'} - dependencies: - ansi-styles: 4.3.0 - astral-regex: 2.0.0 - is-fullwidth-code-point: 3.0.0 - dev: true - /slice-ansi@4.0.0: resolution: {integrity: sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==} engines: {node: '>=10'} @@ -16731,6 +16782,14 @@ packages: is-fullwidth-code-point: 4.0.0 dev: true + /slice-ansi@7.1.0: + resolution: {integrity: sha512-bSiSngZ/jWeX93BqeIAbImyTbEihizcwNjFoRUIY/T1wWQsfsm2Vw1agPKylXvQTU7iASGdHhyqRlqQzfz+Htg==} + engines: {node: '>=18'} + dependencies: + ansi-styles: 6.2.1 + is-fullwidth-code-point: 5.0.0 + dev: true + /snapdragon-node@2.1.1: resolution: {integrity: sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==} engines: {node: '>=0.10.0'} @@ -16987,8 +17046,8 @@ packages: resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} engines: {node: '>=10.0.0'} - /string-argv@0.3.1: - resolution: {integrity: sha512-a1uQGz7IyVy9YwhqjZIZu1c8JO8dNIe20xBmSS6qu9kv++k3JGzCVmprbNN5Kn+BgzD5E7YYwg1CcjuJMRNsvg==} + /string-argv@0.3.2: + resolution: {integrity: sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==} engines: {node: '>=0.6.19'} dev: true @@ -17017,12 +17076,12 @@ packages: strip-ansi: 6.0.1 dev: true - /string-width@5.1.2: - resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} - engines: {node: '>=12'} + /string-width@7.2.0: + resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} + engines: {node: '>=18'} dependencies: - eastasianwidth: 0.2.0 - emoji-regex: 9.2.2 + emoji-regex: 10.4.0 + get-east-asian-width: 1.2.0 strip-ansi: 7.1.0 dev: true @@ -18927,15 +18986,6 @@ packages: workbox-core: 7.0.0 dev: false - /wrap-ansi@6.2.0: - resolution: {integrity: sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==} - engines: {node: '>=8'} - dependencies: - ansi-styles: 4.3.0 - string-width: 4.2.3 - strip-ansi: 6.0.1 - dev: true - /wrap-ansi@7.0.0: resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} engines: {node: '>=10'} @@ -18945,6 +18995,15 @@ packages: strip-ansi: 6.0.1 dev: true + /wrap-ansi@9.0.0: + resolution: {integrity: sha512-G8ura3S+3Z2G+mkgNRq8dqaFZAuxfsxpBB8OCTGRTCtp+l/v9nbFNmCUP1BZMts3G1142MsZfn6eeUKrr4PD1Q==} + engines: {node: '>=18'} + dependencies: + ansi-styles: 6.2.1 + string-width: 7.2.0 + strip-ansi: 7.1.0 + dev: true + /wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} @@ -19056,9 +19115,10 @@ packages: resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} engines: {node: '>= 6'} - /yaml@2.3.1: - resolution: {integrity: sha512-2eHWfjaoXgTBC2jNM1LRef62VQa0umtvRiDSk6HSzW7RvS5YtkabJrwYLLEKWBc8a5U2PTSCs+dJjUTJdlHsWQ==} + /yaml@2.5.1: + resolution: {integrity: sha512-bLQOjaX/ADgQ20isPJRvF0iRUHIxVhYvr53Of7wGcWlO2jvtUlH5m87DsmulFVxRpNLOnI4tB6p/oh8D7kpn9Q==} engines: {node: '>= 14'} + hasBin: true dev: true /yargs-parser@20.2.9: From b24d96e2e87898687d966161818a2b252e32839c Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 7 Oct 2024 14:13:13 +0100 Subject: [PATCH 26/80] EES_5526 - fixed preview token local time issues --- .../ReleaseApiDataSetPreviewTokenPage.tsx | 3 +- tests/robot-tests/tests/libs/utilities.py | 42 ++-- .../public_api/public_api_preview_token.robot | 226 ++++++++++-------- 3 files changed, 158 insertions(+), 113 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx index 57f1afa833a..7d733c9cc10 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx @@ -120,7 +120,8 @@ export default function ReleaseApiDataSetPreviewTokenPage() { {previewToken.expiry} - + {' '} + (local time)

diff --git a/tests/robot-tests/tests/libs/utilities.py b/tests/robot-tests/tests/libs/utilities.py index 0070cbb7e5a..1e8a0dc5817 100644 --- a/tests/robot-tests/tests/libs/utilities.py +++ b/tests/robot-tests/tests/libs/utilities.py @@ -3,20 +3,20 @@ import json import os import re +import time from typing import Union +from urllib.parse import urlparse, urlunparse -import time import pytz import utilities_init import visual from robot.libraries.BuiltIn import BuiltIn +from selenium.common.exceptions import NoSuchElementException from selenium.webdriver.common.by import By from selenium.webdriver.remote.webelement import WebElement -from selenium.common.exceptions import NoSuchElementException from SeleniumLibrary.utils import is_noney from tests.libs.logger import get_logger from tests.libs.selenium_elements import element_finder, sl, waiting -from urllib.parse import urlparse, urlunparse logger = get_logger(__name__) @@ -30,7 +30,6 @@ def _normalize_parent_locator(parent_locator: object) -> Union[str, WebElement]: return parent_locator - def _find_by_label(parent_locator: object, criteria: str, tag: str, constraints: dict) -> list: parent_locator = _normalize_parent_locator(parent_locator) @@ -42,13 +41,11 @@ def _find_by_label(parent_locator: object, criteria: str, tag: str, constraints: for_id = labels[0].get_attribute("for") return get_child_elements(parent_locator, f"id:{for_id}") - def _find_by_testid(parent_locator: object, criteria: str, tag: str, constraints: dict) -> list: parent_locator = _normalize_parent_locator(parent_locator) return get_child_elements(parent_locator, f'css:[data-testid="{criteria}"]') - # Register locator strategies element_finder().register("label", _find_by_label, persist=True) @@ -102,8 +99,13 @@ def retry_or_fail_with_delay(func, retries=5, delay=1.0, *args, **kwargs): def user_waits_until_parent_contains_element( - parent_locator: object, child_locator: str, timeout: int = None, error: str = None, count: int = None, - retries: int = 5, delay: float = 1.0 + parent_locator: object, + child_locator: str, + timeout: int = None, + error: str = None, + count: int = None, + retries: int = 5, + delay: float = 1.0, ): try: child_locator = _normalise_child_locator(child_locator) @@ -290,6 +292,14 @@ def get_current_datetime(strf: str, offset_days: int = 0, timezone: str = "UTC") return format_datetime(datetime.datetime.now(pytz.timezone(timezone)) + datetime.timedelta(days=offset_days), strf) +def get_current_london_datetime(strf: str, offset_days: int = 0) -> str: + return get_current_datetime(strf, offset_days, "Europe/London") + + +def get_current_local_datetime(strf: str, offset_days: int = 0) -> str: + return get_current_datetime(strf, offset_days, _get_browser_timezone()) + + def format_datetime(datetime: datetime, strf: str) -> str: if os.name == "nt": strf = strf.replace("%-", "%#") @@ -297,15 +307,6 @@ def format_datetime(datetime: datetime, strf: str) -> str: return datetime.strftime(strf) -def format_time_without_leading_zero(time_str: str) -> str: - parts = time_str.split() - hour, minute = parts[0].split(':') - - # Remove leading zero in hour - hour = str(int(hour)) - return f"{hour}:{minute} {parts[1]}" - - def user_should_be_at_top_of_page(): (x, y) = sl().get_window_position() if y != 0: @@ -452,7 +453,8 @@ def remove_auth_from_url(publicUrl: str): netloc += f":{parsed_url.port}" modified_url_without_auth = urlunparse( - (parsed_url.scheme, netloc, parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment)) + (parsed_url.scheme, netloc, parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment) + ) return modified_url_without_auth @@ -466,3 +468,7 @@ def get_child_element_with_retry(parent_locator: object, child_locator: str, max logger.warn(f"Child element not found, after ({max_retries}) retries") time.sleep(retry_delay) raise AssertionError(f"Failed to find child element after {max_retries} retries.") + + +def _get_browser_timezone(): + return sl().driver.execute_script("return Intl.DateTimeFormat().resolvedOptions().timeZone;") diff --git a/tests/robot-tests/tests/public_api/public_api_preview_token.robot b/tests/robot-tests/tests/public_api/public_api_preview_token.robot index 5f1ff219c08..a845a6ff5a5 100644 --- a/tests/robot-tests/tests/public_api/public_api_preview_token.robot +++ b/tests/robot-tests/tests/public_api/public_api_preview_token.robot @@ -11,13 +11,15 @@ Suite Setup user signs in as bau1 Suite Teardown user closes the browser Test Setup fail test fast if required + *** Variables *** -${PUBLICATION_NAME}= UI tests - Public API - preview token %{RUN_IDENTIFIER} -${RELEASE_NAME}= Academic year Q1 -${ACADEMIC_YEAR}= 3000 -${SUBJECT_NAME_1}= UI test subject 1 -${SUBJECT_NAME_2}= UI test subject 2 -${PREVIEW_TOKEN_NAME}= Test token +${PUBLICATION_NAME}= UI tests - Public API - preview token %{RUN_IDENTIFIER} +${RELEASE_NAME}= Academic year Q1 +${ACADEMIC_YEAR}= 3000 +${SUBJECT_NAME_1}= UI test subject 1 +${SUBJECT_NAME_2}= UI test subject 2 +${PREVIEW_TOKEN_NAME}= Test token + *** Test Cases *** Create publication and release @@ -33,8 +35,10 @@ Verify release summary user verifies release summary Academic year Q1 3000/01 Accredited official statistics Upload data files - user uploads subject and waits until complete ${SUBJECT_NAME_1} seven_filters.csv seven_filters.meta.csv ${PUBLIC_API_FILES_DIR} - user uploads subject and waits until complete ${SUBJECT_NAME_2} tiny-two-filters.csv tiny-two-filters.meta.csv ${PUBLIC_API_FILES_DIR} + user uploads subject and waits until complete ${SUBJECT_NAME_1} seven_filters.csv seven_filters.meta.csv + ... ${PUBLIC_API_FILES_DIR} + user uploads subject and waits until complete ${SUBJECT_NAME_2} tiny-two-filters.csv + ... tiny-two-filters.meta.csv ${PUBLIC_API_FILES_DIR} Add data guidance to subjects user clicks link Data and files @@ -62,7 +66,7 @@ Create 1st API dataset user clicks button Create API data set ${modal}= user waits until modal is visible Create a new API data set - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_1} + user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_1} user clicks button Confirm new API data set user waits until page finishes loading @@ -76,7 +80,7 @@ Create 2nd API dataset user clicks link Back to API data sets user clicks button Create API data set ${modal}= user waits until modal is visible Create a new API data set - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_2} + user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_2} user clicks button Confirm new API data set user waits until page finishes loading @@ -90,10 +94,11 @@ Verify the contents inside the 'Draft API datasets' table user clicks link Back to API data sets user waits until h3 is visible Draft API data sets - user checks table column heading contains 1 1 Draft version xpath://table[@data-testid='draft-api-data-sets'] - user checks table column heading contains 1 2 Name xpath://table[@data-testid='draft-api-data-sets'] - user checks table column heading contains 1 3 Status xpath://table[@data-testid='draft-api-data-sets'] - user checks table column heading contains 1 4 Actions xpath://table[@data-testid='draft-api-data-sets'] + user checks table column heading contains 1 1 Draft version + ... xpath://table[@data-testid='draft-api-data-sets'] + user checks table column heading contains 1 2 Name xpath://table[@data-testid='draft-api-data-sets'] + user checks table column heading contains 1 3 Status xpath://table[@data-testid='draft-api-data-sets'] + user checks table column heading contains 1 4 Actions xpath://table[@data-testid='draft-api-data-sets'] user checks table cell contains 1 1 v1.0 xpath://table[@data-testid='draft-api-data-sets'] user checks table cell contains 1 3 Ready xpath://table[@data-testid='draft-api-data-sets'] @@ -107,46 +112,67 @@ Click on 'View Details' link(First API dataset) user checks table headings for Draft version details table User checks row data contents inside the 'Draft API datasets' summary table - user checks contents inside the cell value v1.0 xpath://dl[@data-testid="draft-version-summary"]/div/dd[@data-testid='Version-value']/strong - user checks contents inside the cell value Ready xpath:(//div[@data-testid="Status"]//dd[@data-testid="Status-value"]//strong)[2] - user checks contents inside the cell value Academic year Q1 3000/01 xpath:(//div[@data-testid="Release"]//dd[@data-testid="Release-value"]//a)[1] - user checks contents inside the cell value ${SUBJECT_NAME_1} xpath://div[@data-testid="Data set file"]//dd[@data-testid="Data set file-value"] - user checks contents inside the cell value National xpath://div[@data-testid="Geographic levels"]//dd[@data-testid="Geographic levels-value"] - user checks contents inside the cell value 2012/13 xpath://div[@data-testid="Time periods"]//dd[@data-testid="Time periods-value"] - - user checks contents inside the cell value Lower quartile annualised earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[1] - user checks contents inside the cell value Median annualised earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[2] - user checks contents inside the cell value Number of learners with earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[3] - - user clicks button Show 1 more indicator xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"] - - user checks contents inside the cell value Upper quartile annualised earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[4] - - user checks contents inside the cell value Cheese xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[1] - user checks contents inside the cell value Colour xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[2] - user checks contents inside the cell value Ethnicity group xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[3] - - user clicks button Show 4 more filters xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"] - - user checks contents inside the cell value Gender xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[4] - user checks contents inside the cell value Level of learning xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[5] - user checks contents inside the cell value Number of years after achievement of learning aim xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[6] - user checks contents inside the cell value Provision xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[7] - - user checks contents inside the cell value Preview API data set xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[1]/a - user checks contents inside the cell value View preview token log xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[2]/a - user checks contents inside the cell value Remove draft version xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[3]/button + user checks contents inside the cell value v1.0 + ... xpath://dl[@data-testid="draft-version-summary"]/div/dd[@data-testid='Version-value']/strong + user checks contents inside the cell value Ready + ... xpath:(//div[@data-testid="Status"]//dd[@data-testid="Status-value"]//strong)[2] + user checks contents inside the cell value Academic year Q1 3000/01 + ... xpath:(//div[@data-testid="Release"]//dd[@data-testid="Release-value"]//a)[1] + user checks contents inside the cell value ${SUBJECT_NAME_1} + ... xpath://div[@data-testid="Data set file"]//dd[@data-testid="Data set file-value"] + user checks contents inside the cell value National + ... xpath://div[@data-testid="Geographic levels"]//dd[@data-testid="Geographic levels-value"] + user checks contents inside the cell value 2012/13 + ... xpath://div[@data-testid="Time periods"]//dd[@data-testid="Time periods-value"] + + user checks contents inside the cell value Lower quartile annualised earnings + ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[1] + user checks contents inside the cell value Median annualised earnings + ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[2] + user checks contents inside the cell value Number of learners with earnings + ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[3] + + user clicks button Show 1 more indicator + ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"] + + user checks contents inside the cell value Upper quartile annualised earnings + ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[4] + + user checks contents inside the cell value Cheese + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[1] + user checks contents inside the cell value Colour + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[2] + user checks contents inside the cell value Ethnicity group + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[3] + + user clicks button Show 4 more filters xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"] + + user checks contents inside the cell value Gender + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[4] + user checks contents inside the cell value Level of learning + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[5] + user checks contents inside the cell value Number of years after achievement of learning aim + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[6] + user checks contents inside the cell value Provision + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[7] + + user checks contents inside the cell value Preview API data set + ... xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[1]/a + user checks contents inside the cell value View preview token log + ... xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[2]/a + user checks contents inside the cell value Remove draft version + ... xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[3]/button User clicks on 'Preview API data set' link user clicks link containing text Preview API data set User clicks on 'Generate preview token' - user clicks button Generate preview token + user clicks button Generate preview token User creates preview token through 'Generate preview token' modal window ${modal}= user waits until modal is visible Generate preview token user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] ${PREVIEW_TOKEN_NAME} - user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] + user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] user clicks button Continue user waits until page finishes loading @@ -163,12 +189,12 @@ User revokes preview token user waits until page contains Generate API data set preview token User again clicks on 'Generate preview token' - user clicks button Generate preview token + user clicks button Generate preview token User creates another preview token through 'Generate preview token' modal window ${modal}= user waits until modal is visible Generate preview token user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] ${PREVIEW_TOKEN_NAME} - user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] + user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] user clicks button Continue user waits until page finishes loading @@ -188,11 +214,11 @@ User cancels revoking preview token User cancels creating preview token user clicks link Back to API data set details user clicks link containing text Preview API data set - user clicks button Generate preview token + user clicks button Generate preview token ${modal}= user waits until modal is visible Generate preview token user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] ${PREVIEW_TOKEN_NAME} - user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] + user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] user clicks button Cancel user waits until page finishes loading @@ -228,7 +254,7 @@ User verifies the relevant fields on the active preview token page ${modal}= user waits until modal is visible Generate preview token user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] ${PREVIEW_TOKEN_NAME} - user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] + user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] user clicks button Continue user waits until page finishes loading @@ -238,14 +264,9 @@ User verifies the relevant fields on the active preview token page user waits until h2 is visible ${SUBJECT_NAME_1} user checks page contains Reference: ${PREVIEW_TOKEN_NAME} - ${current_time_tomorrow}= get current datetime %Y-%m-%dT%H:%M:%S 1 Europe/London - - ${time_with_leading_zero}= format uk to local datetime ${current_time_tomorrow} %I:%M %p - - ${time_end}= format time without leading zero ${time_with_leading_zero} - + ${current_time_tomorrow}= get current local datetime %-I:%M %p 1 user checks page contains - ... The token expires: tomorrow at ${time_end} + ... The token expires: tomorrow at ${current_time_tomorrow} (local time) user checks page contains button Copy preview token user checks page contains button Revoke preview token @@ -297,55 +318,72 @@ User verifies the row headings and contents in 'Data set details' section user checks row headings within the api data set section Publication user checks row headings within the api data set section Release user checks row headings within the api data set section Release type - user checks row headings within the api data set section Geographic levels - user checks row headings within the api data set section Indicators + user checks row headings within the api data set section Geographic levels + user checks row headings within the api data set section Indicators user checks row headings within the api data set section Filters user checks row headings within the api data set section Time period - user checks row headings within the api data set section Notifications - - user checks contents inside the cell value Test theme css: #dataSetDetails [data-testid="Theme-value"] - user checks contents inside the cell value ${PUBLICATION_NAME} css:#dataSetDetails [data-testid="Publication-value"] - user checks contents inside the cell value Academic year Q1 3000/01 css:#dataSetDetails [data-testid="Release-value"] - User checks contents inside the release type Accredited official statistics css:#dataSetDetails [data-testid="Release type-value"] > button - user checks contents inside the cell value National css:#dataSetDetails [data-testid="Geographic levels-value"] - - user checks contents inside the cell value Lower quartile annualised earnings css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(1) - user checks contents inside the cell value Median annualised earnings css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(2) - user checks contents inside the cell value Number of learners with earnings css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(3) - - user clicks button Show 1 more indicator css:#dataSetDetails [data-testid="Indicators-value"] - - user checks contents inside the cell value Upper quartile annualised earnings css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(4) - - user checks contents inside the cell value Cheese css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(1) - user checks contents inside the cell value Colour css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(2) - user checks contents inside the cell value Ethnicity group css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(3) - - user clicks button Show 4 more filters css:#dataSetDetails [data-testid="Filters-value"] - - user checks contents inside the cell value Gender css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(4) - user checks contents inside the cell value Level of learning css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(5) - user checks contents inside the cell value Number of years after achievement of learning aim css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(6) - user checks contents inside the cell value Provision css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(7) - - user checks contents inside the cell value 2012/13 css:#dataSetDetails [data-testid="Time period-value"] - user checks contents inside the cell value Get email updates about this API data set css:#dataSetDetails [data-testid="Notifications-value"] > a + user checks row headings within the api data set section Notifications + + user checks contents inside the cell value Test theme css: #dataSetDetails [data-testid="Theme-value"] + user checks contents inside the cell value ${PUBLICATION_NAME} + ... css:#dataSetDetails [data-testid="Publication-value"] + user checks contents inside the cell value Academic year Q1 3000/01 + ... css:#dataSetDetails [data-testid="Release-value"] + User checks contents inside the release type Accredited official statistics + ... css:#dataSetDetails [data-testid="Release type-value"] > button + user checks contents inside the cell value National + ... css:#dataSetDetails [data-testid="Geographic levels-value"] + + user checks contents inside the cell value Lower quartile annualised earnings + ... css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(1) + user checks contents inside the cell value Median annualised earnings + ... css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(2) + user checks contents inside the cell value Number of learners with earnings + ... css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(3) + + user clicks button Show 1 more indicator css:#dataSetDetails [data-testid="Indicators-value"] + + user checks contents inside the cell value Upper quartile annualised earnings + ... css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(4) + + user checks contents inside the cell value Cheese + ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(1) + user checks contents inside the cell value Colour + ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(2) + user checks contents inside the cell value Ethnicity group + ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(3) + + user clicks button Show 4 more filters css:#dataSetDetails [data-testid="Filters-value"] + + user checks contents inside the cell value Gender + ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(4) + user checks contents inside the cell value Level of learning + ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(5) + user checks contents inside the cell value Number of years after achievement of learning aim + ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(6) + user checks contents inside the cell value Provision + ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(7) + + user checks contents inside the cell value 2012/13 css:#dataSetDetails [data-testid="Time period-value"] + user checks contents inside the cell value Get email updates about this API data set + ... css:#dataSetDetails [data-testid="Notifications-value"] > a User verifies the headings and contents in 'API version history' section user checks table column heading contains 1 1 Version css:section[id="apiVersionHistory"] user checks table column heading contains 1 2 Release css:section[id="apiVersionHistory"] - user checks table column heading contains 1 3 Status css:section[id="apiVersionHistory"] + user checks table column heading contains 1 3 Status css:section[id="apiVersionHistory"] + + user checks table cell contains 1 1 1.0 (current) xpath://section[@id="apiVersionHistory"] + user checks table cell contains 1 2 Academic year Q1 3000/01 xpath://section[@id="apiVersionHistory"] + user checks table cell contains 1 3 Published xpath://section[@id="apiVersionHistory"] - user checks table cell contains 1 1 1.0 (current) xpath://section[@id="apiVersionHistory"] - user checks table cell contains 1 2 Academic year Q1 3000/01 xpath://section[@id="apiVersionHistory"] - user checks table cell contains 1 3 Published xpath://section[@id="apiVersionHistory"] *** Keywords *** User checks contents inside the release type - [Arguments] ${expected_text} ${locator} - ${full_text}= get text ${locator} + [Arguments] ${expected_text} ${locator} + ${full_text}= get text ${locator} # Split and remove the part after '?' and strip whitespace ${button_text}= set variable ${full_text.split('?')[0].strip()} From f4b7039df1ce42e1daa909a22fbe46f82195a61e Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 7 Oct 2024 14:15:56 +0100 Subject: [PATCH 27/80] EES_5526 - fixed preview token local time issues --- .../ReleaseApiDataSetPreviewTokenPage.tsx | 20 +- tests/robot-tests/tests/libs/utilities.py | 23 +- .../public_api/public_api_preview_token.robot | 219 +++++++----------- 3 files changed, 106 insertions(+), 156 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx index 7d733c9cc10..19c46a479ce 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx @@ -1,5 +1,5 @@ import Link from '@admin/components/Link'; -import { useConfig } from '@admin/contexts/ConfigContext'; +import {useConfig} from '@admin/contexts/ConfigContext'; import { releaseApiDataSetDetailsRoute, releaseApiDataSetPreviewRoute, @@ -10,7 +10,7 @@ import { import previewTokenQueries from '@admin/queries/previewTokenQueries'; import apiDataSetQueries from '@admin/queries/apiDataSetQueries'; import previewTokenService from '@admin/services/previewTokenService'; -import { useLastLocation } from '@admin/contexts/LastLocationContext'; +import {useLastLocation} from '@admin/contexts/LastLocationContext'; import CodeBlock from '@common/components/CodeBlock'; import LoadingSpinner from '@common/components/LoadingSpinner'; import Button from '@common/components/Button'; @@ -20,24 +20,24 @@ import ModalConfirm from '@common/components/ModalConfirm'; import Tabs from '@common/components/Tabs'; import TabsSection from '@common/components/TabsSection'; import ApiDataSetQuickStart from '@common/modules/data-catalogue/components/ApiDataSetQuickStart'; -import { useQuery } from '@tanstack/react-query'; -import { generatePath, useHistory, useParams } from 'react-router-dom'; +import {useQuery} from '@tanstack/react-query'; +import {generatePath, useHistory, useParams} from 'react-router-dom'; import React from 'react'; export default function ReleaseApiDataSetPreviewTokenPage() { const history = useHistory(); const lastLocation = useLastLocation(); - const { publicApiUrl, publicApiDocsUrl } = useConfig(); + const {publicApiUrl, publicApiDocsUrl} = useConfig(); - const { dataSetId, previewTokenId, releaseId, publicationId } = + const {dataSetId, previewTokenId, releaseId, publicationId} = useParams(); - const { data: dataSet, isLoading: isLoadingDataSet } = useQuery( + const {data: dataSet, isLoading: isLoadingDataSet} = useQuery( apiDataSetQueries.get(dataSetId), ); - const { data: previewToken, isLoading: isLoadingPreviewTokenId } = useQuery({ + const {data: previewToken, isLoading: isLoadingPreviewTokenId} = useQuery({ ...previewTokenQueries.get(previewTokenId), }); @@ -120,8 +120,8 @@ export default function ReleaseApiDataSetPreviewTokenPage() { {previewToken.expiry} - {' '} - (local time) + + {' '}(local time)

diff --git a/tests/robot-tests/tests/libs/utilities.py b/tests/robot-tests/tests/libs/utilities.py index 1e8a0dc5817..6b1d449da3d 100644 --- a/tests/robot-tests/tests/libs/utilities.py +++ b/tests/robot-tests/tests/libs/utilities.py @@ -3,20 +3,20 @@ import json import os import re -import time from typing import Union -from urllib.parse import urlparse, urlunparse +import time import pytz import utilities_init import visual from robot.libraries.BuiltIn import BuiltIn -from selenium.common.exceptions import NoSuchElementException from selenium.webdriver.common.by import By from selenium.webdriver.remote.webelement import WebElement +from selenium.common.exceptions import NoSuchElementException from SeleniumLibrary.utils import is_noney from tests.libs.logger import get_logger from tests.libs.selenium_elements import element_finder, sl, waiting +from urllib.parse import urlparse, urlunparse logger = get_logger(__name__) @@ -99,13 +99,8 @@ def retry_or_fail_with_delay(func, retries=5, delay=1.0, *args, **kwargs): def user_waits_until_parent_contains_element( - parent_locator: object, - child_locator: str, - timeout: int = None, - error: str = None, - count: int = None, - retries: int = 5, - delay: float = 1.0, + parent_locator: object, child_locator: str, timeout: int = None, error: str = None, count: int = None, + retries: int = 5, delay: float = 1.0 ): try: child_locator = _normalise_child_locator(child_locator) @@ -293,7 +288,7 @@ def get_current_datetime(strf: str, offset_days: int = 0, timezone: str = "UTC") def get_current_london_datetime(strf: str, offset_days: int = 0) -> str: - return get_current_datetime(strf, offset_days, "Europe/London") + return get_current_datetime(strf, offset_days, 'Europe/London') def get_current_local_datetime(strf: str, offset_days: int = 0) -> str: @@ -453,8 +448,7 @@ def remove_auth_from_url(publicUrl: str): netloc += f":{parsed_url.port}" modified_url_without_auth = urlunparse( - (parsed_url.scheme, netloc, parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment) - ) + (parsed_url.scheme, netloc, parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment)) return modified_url_without_auth @@ -469,6 +463,5 @@ def get_child_element_with_retry(parent_locator: object, child_locator: str, max time.sleep(retry_delay) raise AssertionError(f"Failed to find child element after {max_retries} retries.") - def _get_browser_timezone(): - return sl().driver.execute_script("return Intl.DateTimeFormat().resolvedOptions().timeZone;") + return sl().driver.execute_script('return Intl.DateTimeFormat().resolvedOptions().timeZone;') diff --git a/tests/robot-tests/tests/public_api/public_api_preview_token.robot b/tests/robot-tests/tests/public_api/public_api_preview_token.robot index a845a6ff5a5..325aae7bad6 100644 --- a/tests/robot-tests/tests/public_api/public_api_preview_token.robot +++ b/tests/robot-tests/tests/public_api/public_api_preview_token.robot @@ -11,15 +11,13 @@ Suite Setup user signs in as bau1 Suite Teardown user closes the browser Test Setup fail test fast if required - *** Variables *** -${PUBLICATION_NAME}= UI tests - Public API - preview token %{RUN_IDENTIFIER} -${RELEASE_NAME}= Academic year Q1 -${ACADEMIC_YEAR}= 3000 -${SUBJECT_NAME_1}= UI test subject 1 -${SUBJECT_NAME_2}= UI test subject 2 -${PREVIEW_TOKEN_NAME}= Test token - +${PUBLICATION_NAME}= UI tests - Public API - preview token %{RUN_IDENTIFIER} +${RELEASE_NAME}= Academic year Q1 +${ACADEMIC_YEAR}= 3000 +${SUBJECT_NAME_1}= UI test subject 1 +${SUBJECT_NAME_2}= UI test subject 2 +${PREVIEW_TOKEN_NAME}= Test token *** Test Cases *** Create publication and release @@ -35,10 +33,8 @@ Verify release summary user verifies release summary Academic year Q1 3000/01 Accredited official statistics Upload data files - user uploads subject and waits until complete ${SUBJECT_NAME_1} seven_filters.csv seven_filters.meta.csv - ... ${PUBLIC_API_FILES_DIR} - user uploads subject and waits until complete ${SUBJECT_NAME_2} tiny-two-filters.csv - ... tiny-two-filters.meta.csv ${PUBLIC_API_FILES_DIR} + user uploads subject and waits until complete ${SUBJECT_NAME_1} seven_filters.csv seven_filters.meta.csv ${PUBLIC_API_FILES_DIR} + user uploads subject and waits until complete ${SUBJECT_NAME_2} tiny-two-filters.csv tiny-two-filters.meta.csv ${PUBLIC_API_FILES_DIR} Add data guidance to subjects user clicks link Data and files @@ -66,7 +62,7 @@ Create 1st API dataset user clicks button Create API data set ${modal}= user waits until modal is visible Create a new API data set - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_1} + user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_1} user clicks button Confirm new API data set user waits until page finishes loading @@ -80,7 +76,7 @@ Create 2nd API dataset user clicks link Back to API data sets user clicks button Create API data set ${modal}= user waits until modal is visible Create a new API data set - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_2} + user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_2} user clicks button Confirm new API data set user waits until page finishes loading @@ -94,11 +90,10 @@ Verify the contents inside the 'Draft API datasets' table user clicks link Back to API data sets user waits until h3 is visible Draft API data sets - user checks table column heading contains 1 1 Draft version - ... xpath://table[@data-testid='draft-api-data-sets'] - user checks table column heading contains 1 2 Name xpath://table[@data-testid='draft-api-data-sets'] - user checks table column heading contains 1 3 Status xpath://table[@data-testid='draft-api-data-sets'] - user checks table column heading contains 1 4 Actions xpath://table[@data-testid='draft-api-data-sets'] + user checks table column heading contains 1 1 Draft version xpath://table[@data-testid='draft-api-data-sets'] + user checks table column heading contains 1 2 Name xpath://table[@data-testid='draft-api-data-sets'] + user checks table column heading contains 1 3 Status xpath://table[@data-testid='draft-api-data-sets'] + user checks table column heading contains 1 4 Actions xpath://table[@data-testid='draft-api-data-sets'] user checks table cell contains 1 1 v1.0 xpath://table[@data-testid='draft-api-data-sets'] user checks table cell contains 1 3 Ready xpath://table[@data-testid='draft-api-data-sets'] @@ -112,67 +107,46 @@ Click on 'View Details' link(First API dataset) user checks table headings for Draft version details table User checks row data contents inside the 'Draft API datasets' summary table - user checks contents inside the cell value v1.0 - ... xpath://dl[@data-testid="draft-version-summary"]/div/dd[@data-testid='Version-value']/strong - user checks contents inside the cell value Ready - ... xpath:(//div[@data-testid="Status"]//dd[@data-testid="Status-value"]//strong)[2] - user checks contents inside the cell value Academic year Q1 3000/01 - ... xpath:(//div[@data-testid="Release"]//dd[@data-testid="Release-value"]//a)[1] - user checks contents inside the cell value ${SUBJECT_NAME_1} - ... xpath://div[@data-testid="Data set file"]//dd[@data-testid="Data set file-value"] - user checks contents inside the cell value National - ... xpath://div[@data-testid="Geographic levels"]//dd[@data-testid="Geographic levels-value"] - user checks contents inside the cell value 2012/13 - ... xpath://div[@data-testid="Time periods"]//dd[@data-testid="Time periods-value"] - - user checks contents inside the cell value Lower quartile annualised earnings - ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[1] - user checks contents inside the cell value Median annualised earnings - ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[2] - user checks contents inside the cell value Number of learners with earnings - ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[3] - - user clicks button Show 1 more indicator - ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"] - - user checks contents inside the cell value Upper quartile annualised earnings - ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[4] - - user checks contents inside the cell value Cheese - ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[1] - user checks contents inside the cell value Colour - ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[2] - user checks contents inside the cell value Ethnicity group - ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[3] - - user clicks button Show 4 more filters xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"] - - user checks contents inside the cell value Gender - ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[4] - user checks contents inside the cell value Level of learning - ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[5] - user checks contents inside the cell value Number of years after achievement of learning aim - ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[6] - user checks contents inside the cell value Provision - ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[7] - - user checks contents inside the cell value Preview API data set - ... xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[1]/a - user checks contents inside the cell value View preview token log - ... xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[2]/a - user checks contents inside the cell value Remove draft version - ... xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[3]/button + user checks contents inside the cell value v1.0 xpath://dl[@data-testid="draft-version-summary"]/div/dd[@data-testid='Version-value']/strong + user checks contents inside the cell value Ready xpath:(//div[@data-testid="Status"]//dd[@data-testid="Status-value"]//strong)[2] + user checks contents inside the cell value Academic year Q1 3000/01 xpath:(//div[@data-testid="Release"]//dd[@data-testid="Release-value"]//a)[1] + user checks contents inside the cell value ${SUBJECT_NAME_1} xpath://div[@data-testid="Data set file"]//dd[@data-testid="Data set file-value"] + user checks contents inside the cell value National xpath://div[@data-testid="Geographic levels"]//dd[@data-testid="Geographic levels-value"] + user checks contents inside the cell value 2012/13 xpath://div[@data-testid="Time periods"]//dd[@data-testid="Time periods-value"] + + user checks contents inside the cell value Lower quartile annualised earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[1] + user checks contents inside the cell value Median annualised earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[2] + user checks contents inside the cell value Number of learners with earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[3] + + user clicks button Show 1 more indicator xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"] + + user checks contents inside the cell value Upper quartile annualised earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[4] + + user checks contents inside the cell value Cheese xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[1] + user checks contents inside the cell value Colour xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[2] + user checks contents inside the cell value Ethnicity group xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[3] + + user clicks button Show 4 more filters xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"] + + user checks contents inside the cell value Gender xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[4] + user checks contents inside the cell value Level of learning xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[5] + user checks contents inside the cell value Number of years after achievement of learning aim xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[6] + user checks contents inside the cell value Provision xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[7] + + user checks contents inside the cell value Preview API data set xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[1]/a + user checks contents inside the cell value View preview token log xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[2]/a + user checks contents inside the cell value Remove draft version xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[3]/button User clicks on 'Preview API data set' link user clicks link containing text Preview API data set User clicks on 'Generate preview token' - user clicks button Generate preview token + user clicks button Generate preview token User creates preview token through 'Generate preview token' modal window ${modal}= user waits until modal is visible Generate preview token user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] ${PREVIEW_TOKEN_NAME} - user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] + user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] user clicks button Continue user waits until page finishes loading @@ -189,12 +163,12 @@ User revokes preview token user waits until page contains Generate API data set preview token User again clicks on 'Generate preview token' - user clicks button Generate preview token + user clicks button Generate preview token User creates another preview token through 'Generate preview token' modal window ${modal}= user waits until modal is visible Generate preview token user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] ${PREVIEW_TOKEN_NAME} - user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] + user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] user clicks button Continue user waits until page finishes loading @@ -214,11 +188,11 @@ User cancels revoking preview token User cancels creating preview token user clicks link Back to API data set details user clicks link containing text Preview API data set - user clicks button Generate preview token + user clicks button Generate preview token ${modal}= user waits until modal is visible Generate preview token user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] ${PREVIEW_TOKEN_NAME} - user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] + user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] user clicks button Cancel user waits until page finishes loading @@ -254,7 +228,7 @@ User verifies the relevant fields on the active preview token page ${modal}= user waits until modal is visible Generate preview token user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] ${PREVIEW_TOKEN_NAME} - user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] + user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] user clicks button Continue user waits until page finishes loading @@ -266,7 +240,7 @@ User verifies the relevant fields on the active preview token page ${current_time_tomorrow}= get current local datetime %-I:%M %p 1 user checks page contains - ... The token expires: tomorrow at ${current_time_tomorrow} (local time) + ... The token expires: tomorrow at ${current_time_tomorrow} (local time) user checks page contains button Copy preview token user checks page contains button Revoke preview token @@ -318,72 +292,55 @@ User verifies the row headings and contents in 'Data set details' section user checks row headings within the api data set section Publication user checks row headings within the api data set section Release user checks row headings within the api data set section Release type - user checks row headings within the api data set section Geographic levels - user checks row headings within the api data set section Indicators + user checks row headings within the api data set section Geographic levels + user checks row headings within the api data set section Indicators user checks row headings within the api data set section Filters user checks row headings within the api data set section Time period - user checks row headings within the api data set section Notifications - - user checks contents inside the cell value Test theme css: #dataSetDetails [data-testid="Theme-value"] - user checks contents inside the cell value ${PUBLICATION_NAME} - ... css:#dataSetDetails [data-testid="Publication-value"] - user checks contents inside the cell value Academic year Q1 3000/01 - ... css:#dataSetDetails [data-testid="Release-value"] - User checks contents inside the release type Accredited official statistics - ... css:#dataSetDetails [data-testid="Release type-value"] > button - user checks contents inside the cell value National - ... css:#dataSetDetails [data-testid="Geographic levels-value"] - - user checks contents inside the cell value Lower quartile annualised earnings - ... css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(1) - user checks contents inside the cell value Median annualised earnings - ... css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(2) - user checks contents inside the cell value Number of learners with earnings - ... css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(3) - - user clicks button Show 1 more indicator css:#dataSetDetails [data-testid="Indicators-value"] - - user checks contents inside the cell value Upper quartile annualised earnings - ... css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(4) - - user checks contents inside the cell value Cheese - ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(1) - user checks contents inside the cell value Colour - ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(2) - user checks contents inside the cell value Ethnicity group - ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(3) - - user clicks button Show 4 more filters css:#dataSetDetails [data-testid="Filters-value"] - - user checks contents inside the cell value Gender - ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(4) - user checks contents inside the cell value Level of learning - ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(5) - user checks contents inside the cell value Number of years after achievement of learning aim - ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(6) - user checks contents inside the cell value Provision - ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(7) - - user checks contents inside the cell value 2012/13 css:#dataSetDetails [data-testid="Time period-value"] - user checks contents inside the cell value Get email updates about this API data set - ... css:#dataSetDetails [data-testid="Notifications-value"] > a + user checks row headings within the api data set section Notifications + + user checks contents inside the cell value Test theme css: #dataSetDetails [data-testid="Theme-value"] + user checks contents inside the cell value ${PUBLICATION_NAME} css:#dataSetDetails [data-testid="Publication-value"] + user checks contents inside the cell value Academic year Q1 3000/01 css:#dataSetDetails [data-testid="Release-value"] + User checks contents inside the release type Accredited official statistics css:#dataSetDetails [data-testid="Release type-value"] > button + user checks contents inside the cell value National css:#dataSetDetails [data-testid="Geographic levels-value"] + + user checks contents inside the cell value Lower quartile annualised earnings css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(1) + user checks contents inside the cell value Median annualised earnings css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(2) + user checks contents inside the cell value Number of learners with earnings css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(3) + + user clicks button Show 1 more indicator css:#dataSetDetails [data-testid="Indicators-value"] + + user checks contents inside the cell value Upper quartile annualised earnings css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(4) + + user checks contents inside the cell value Cheese css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(1) + user checks contents inside the cell value Colour css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(2) + user checks contents inside the cell value Ethnicity group css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(3) + + user clicks button Show 4 more filters css:#dataSetDetails [data-testid="Filters-value"] + + user checks contents inside the cell value Gender css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(4) + user checks contents inside the cell value Level of learning css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(5) + user checks contents inside the cell value Number of years after achievement of learning aim css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(6) + user checks contents inside the cell value Provision css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(7) + + user checks contents inside the cell value 2012/13 css:#dataSetDetails [data-testid="Time period-value"] + user checks contents inside the cell value Get email updates about this API data set css:#dataSetDetails [data-testid="Notifications-value"] > a User verifies the headings and contents in 'API version history' section user checks table column heading contains 1 1 Version css:section[id="apiVersionHistory"] user checks table column heading contains 1 2 Release css:section[id="apiVersionHistory"] - user checks table column heading contains 1 3 Status css:section[id="apiVersionHistory"] - - user checks table cell contains 1 1 1.0 (current) xpath://section[@id="apiVersionHistory"] - user checks table cell contains 1 2 Academic year Q1 3000/01 xpath://section[@id="apiVersionHistory"] - user checks table cell contains 1 3 Published xpath://section[@id="apiVersionHistory"] + user checks table column heading contains 1 3 Status css:section[id="apiVersionHistory"] + user checks table cell contains 1 1 1.0 (current) xpath://section[@id="apiVersionHistory"] + user checks table cell contains 1 2 Academic year Q1 3000/01 xpath://section[@id="apiVersionHistory"] + user checks table cell contains 1 3 Published xpath://section[@id="apiVersionHistory"] *** Keywords *** User checks contents inside the release type - [Arguments] ${expected_text} ${locator} - ${full_text}= get text ${locator} + [Arguments] ${expected_text} ${locator} + ${full_text}= get text ${locator} # Split and remove the part after '?' and strip whitespace ${button_text}= set variable ${full_text.split('?')[0].strip()} From 2e23cbd4741ec38b7917b74f28f3e311b5456870 Mon Sep 17 00:00:00 2001 From: "rian.thwaite" Date: Mon, 7 Oct 2024 15:54:30 +0100 Subject: [PATCH 28/80] EES-5527 edit title key validation bug fix --- .../content/components/EditableKeyStatDataBlock.tsx | 8 +++++--- .../release/content/components/EditableKeyStatText.tsx | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatDataBlock.tsx b/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatDataBlock.tsx index b5119a4d72e..7d14c244047 100644 --- a/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatDataBlock.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatDataBlock.tsx @@ -10,7 +10,7 @@ import useToggle from '@common/hooks/useToggle'; import tableBuilderQueries from '@common/modules/find-statistics/queries/tableBuilderQueries'; import { KeyStatisticDataBlock } from '@common/services/publicationService'; import { useQuery } from '@tanstack/react-query'; -import React, { useCallback } from 'react'; +import React, { useCallback, useState } from 'react'; export interface EditableKeyStatDataBlockProps { isEditing?: boolean; @@ -33,6 +33,7 @@ export default function EditableKeyStatDataBlock({ onRemove, onSubmit, }: EditableKeyStatDataBlockProps) { + const [keyStatisticId, setKeyStatisticId] = useState(""); const [showForm, toggleShowForm] = useToggle(false); const { @@ -46,9 +47,10 @@ export default function EditableKeyStatDataBlock({ const handleSubmit = useCallback( async (values: KeyStatDataBlockFormValues) => { await onSubmit(values); + setKeyStatisticId(""); toggleShowForm.off(); }, - [onSubmit, toggleShowForm], + [onSubmit, toggleShowForm, setKeyStatisticId], ); if (isLoading) { @@ -76,7 +78,7 @@ export default function EditableKeyStatDataBlock({ return ( keyStatTitle === keyStatisticId)} title={title} statistic={statistic} isReordering={isReordering} diff --git a/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatText.tsx b/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatText.tsx index 652cd544bcb..8c963baad8b 100644 --- a/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatText.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatText.tsx @@ -4,7 +4,7 @@ import EditableKeyStatTextForm, { } from '@admin/pages/release/content/components/EditableKeyStatTextForm'; import useToggle from '@common/hooks/useToggle'; import { KeyStatisticText } from '@common/services/publicationService'; -import React, { useCallback } from 'react'; +import React, { useCallback, useState } from 'react'; export interface EditableKeyStatTextProps { isEditing?: boolean; @@ -25,21 +25,23 @@ export default function EditableKeyStatText({ onRemove, onSubmit, }: EditableKeyStatTextProps) { + const [keyStatisticId, setKeyStatisticId] = useState(""); const [showForm, toggleShowForm] = useToggle(false); const handleSubmit = useCallback( async (values: KeyStatTextFormValues) => { await onSubmit(values); + setKeyStatisticId(""); toggleShowForm.off(); }, - [onSubmit, toggleShowForm], + [onSubmit, setKeyStatisticId, toggleShowForm], ); if (showForm) { return ( keyStatTitle === keyStatisticId)} isReordering={isReordering} testId={testId} onSubmit={handleSubmit} From 597b8f2319e7e10d9a31b193ef80099f4ce0bdc6 Mon Sep 17 00:00:00 2001 From: Ben Outram Date: Mon, 7 Oct 2024 17:13:59 +0100 Subject: [PATCH 29/80] EES-5413 Generate data set version changes while processing completion of data set version mappings (#5227) * EES-5413 [WIP] Adding new step to the processor to generate the changelog * EES-5413 [WIP] Adding the first test for location changelogs * EES-5413 Refactoring out setup logic from test * EES-5413 Adding tests for the filter changelogs * EES-5413 Code tidy-up * EES-5413 Removing a comment * EES-5413 Updating for recent changes to MappingKeyGenerators * EES-5413 Fix expectedActivitySequence in ProcessCompletionOfNextDataSetVersionImportTests.Success test. * EES-5413 Rename DataSetVersionChangelogService to DataSetVersionChangeService. * EES-5413 Reformatting * EES-5413 Tidy-up DataSetVersionChangeService * EES-5413 Refactor CreateDataSet method to CreateDataSetAndInitialVersion and add new method CreateDataSetAndInitialVersionAndNextVersion * EES-5419 Add CreateChangesTestsTimePeriodTests * EES-5413 Add CreateChangesTestsGeographicLevelTests * EES-5413 Add CreateChangesTestsIndicatorTests * EES-5413 Remove method GetVersionWithChanges in favour of getting version with specific includes * EES-5413 Addressing PR comments * EES-5413 [WIP] - Refactoring to not use mappings for determining changes * EES-5413 Refactoring Filter & Location tests * EES-5413 Allowing updates to indicators, where the Public ID hasn't changed i.e. changes to the column name, labels, units, decimal places. * EES-5413 Ordering changes sensibly (by label for filters etc.) before inserting them into the DB * EES-5413 Fixing how filter changes are ordered before they're inserted into the DB + added an integration test for it * EES-5413 Tidying up the filter changes ordering tests * EES-5413 Removing unnecessary tuple usage * EES-5413 Fixing the ordering of location changes when they're inserted into the DB + Adding integration tests for the ordering * EES-5413 Fixing Indicator changes ordering when inserting them into the DB + Adding integration test for it * EES-5413 Removing code which is no longer necessary * EES-5413 Removing more unused code * EES-5413 Simplifying test assertions * EES-5413 Replacing all sqids with the `SqidEncoder.Encode(num)` method call, to make the tests easier to reason * EES-5413 Formatting all test code * EES-5413 Simplifying service logic & fixing bug whereby we were checking for location updates incorrectly * EES-5413 Test tidy-up * EES-5413 Removing redundant test case * EES-5413 No longer storing changelogs for newly ADDED OPTIONS when they are for a newly ADDED FILTER/LOCATION --------- Co-authored-by: jack-hive <148866614+jack-hive@users.noreply.github.com> --- .../Extensions/EnumerableExtensionsTests.cs | 646 ++-- .../Extensions/EnumerableExtensions.cs | 23 + .../FilterMappingPlanGeneratorExtensions.cs | 22 +- .../GeographicLevelMetaGeneratorExtensions.cs | 2 +- .../LocationMappingPlanGeneratorExtensions.cs | 12 +- .../Change.cs | 24 + .../DataSetVersionImportStage.cs | 1 + .../TimePeriodMeta.cs | 15 + .../Functions/CopyCsvFilesFunctionTests.cs | 2 +- .../HandleProcessingFailureFunctionTests.cs | 2 +- .../Functions/ImportDataFunctionTests.cs | 14 +- .../Functions/ImportMetadataFunctionTests.cs | 40 +- ...OfNextDataSetVersionImportFunctionTests.cs | 3000 ++++++++++++++++- ...ocessInitialDataSetVersionFunctionTests.cs | 4 +- ...NextDataSetVersionMappingsFunctionTests.cs | 13 +- .../Functions/WriteDataFilesFunctionTests.cs | 2 +- .../Models/DataSetVersionMeta.cs | 16 + .../ProcessorFunctionsIntegrationTest.cs | 108 +- .../Functions/ActivityNames.cs | 2 + ...sCompletionOfNextDataSetVersionFunction.cs | 15 +- .../ProcessorHostBuilder.cs | 1 + .../Services/DataSetVersionChangeService.cs | 507 +++ .../IDataSetVersionChangeService.cs | 8 + 23 files changed, 4147 insertions(+), 332 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Models/DataSetVersionMeta.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionChangeService.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetVersionChangeService.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EnumerableExtensionsTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EnumerableExtensionsTests.cs index 938d1120108..08917f36617 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EnumerableExtensionsTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/EnumerableExtensionsTests.cs @@ -5,304 +5,486 @@ using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; -using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using Xunit; -using static GovUk.Education.ExploreEducationStatistics.Common.Services.CollectionUtils; -namespace GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions +namespace GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; + +public class EnumerableExtensionsTests { - public class EnumerableExtensionsTests + public class NaturalOrderByTests { [Fact] - public void ToDictionaryIndexed() + public void OrdersSingleCharacters() { - var source = new List - { - "a", - "b", - "c" - }; - - var result = source.ToDictionaryIndexed(value => value, (value, index) => (Value: value, Index: index)); - - var expected = new Dictionary - { - {"a", ("a", 0)}, - {"b", ("b", 1)}, - {"c", ("c", 2)} - }; - - Assert.Equal(expected, result); + TestRecord[] testRecords = [ + new() { Value = "d" }, + new() { Value = "1" }, + new() { Value = "b" }, + new() { Value = "c" }, + new() { Value = "a" }, + ]; + + TestRecord[] expected = [ + new() { Value = "1" }, + new() { Value = "a" }, + new() { Value = "b" }, + new() { Value = "c" }, + new() { Value = "d" }, + ]; + + var orderedRecords = testRecords + .NaturalOrderBy(r => r.Value); + + Assert.Equal(expected, orderedRecords); } [Fact] - public void DistinctByProperty() + public void OrdersMultipleCharacters() { - var list = new List - { - new(1), - new(1), - new(2), - }; - - var distinct = list.DistinctByProperty(x => x.Value).ToList(); - - Assert.Equal(2, distinct.Count); - Assert.Equal(1, distinct[0].Value); - Assert.Equal(2, distinct[1].Value); + TestRecord[] testRecords = [ + new() { Value = "z24" }, + new() { Value = "z2" }, + new() { Value = "abcde" }, + new() { Value = "z15" }, + new() { Value = "abd" }, + new() { Value = "z1" }, + new() { Value = "z3" }, + new() { Value = "abc" }, + new() { Value = "z20" }, + new() { Value = "z5" }, + new() { Value = "abcd" }, + new() { Value = "z11" }, + new() { Value = "z22" }, + ]; + + TestRecord[] expected = [ + new() { Value = "abc" }, + new() { Value = "abcd" }, + new() { Value = "abcde" }, + new() { Value = "abd" }, + new() { Value = "z1" }, + new() { Value = "z2" }, + new() { Value = "z3" }, + new() { Value = "z5" }, + new() { Value = "z11" }, + new() { Value = "z15" }, + new() { Value = "z20" }, + new() { Value = "z22" }, + new() { Value = "z24" }, + ]; + + var orderedRecords = testRecords + .NaturalOrderBy(r => r.Value); + + Assert.Equal(expected, orderedRecords); } - [Fact] - public void IndexOfFirst() + private record TestRecord { - var result = new List { 1, 2, 3 } - .IndexOfFirst(value => value == 2); - - Assert.Equal(1, result); + public required string Value { get; init; } } + } + public class NaturalThenByTests + { [Fact] - public void IndexOfFirst_NoMatch() + public void OrdersSingleCharacters() { - var result = new List { 1, 2, 3 } - .IndexOfFirst(value => value == 4); - - Assert.Equal(-1, result); + TestRecord[] testRecords = + [ + new() { FirstString = "1", SecondString = "d" }, + new() { FirstString = "1", SecondString = "1" }, + new() { FirstString = "1", SecondString = "b" }, + new() { FirstString = "1", SecondString = "c" }, + new() { FirstString = "1", SecondString = "a" }, + ]; + + TestRecord[] expected = + [ + new() { FirstString = "1", SecondString = "1" }, + new() { FirstString = "1", SecondString = "a" }, + new() { FirstString = "1", SecondString = "b" }, + new() { FirstString = "1", SecondString = "c" }, + new() { FirstString = "1", SecondString = "d" }, + ]; + + var orderedRecords = testRecords + .NaturalOrderBy(r => r.FirstString) + .NaturalThenBy(r => r.SecondString); + + Assert.Equal(expected, orderedRecords); } [Fact] - public async Task ForEachAsync_SuccessfulEitherList() + public void OrdersMultipleCharacters() { - Either> results = - await new List {1, 2} - .ForEachAsync(async value => - await GetSuccessfulEither(value)); - - Assert.True(results.IsRight); - - var resultsList = results.Right; - Assert.Equal(2, resultsList.Count); - Assert.Contains(1, resultsList); - Assert.Contains(2, resultsList); + TestRecord[] testRecords = + [ + new() { FirstString = "1", SecondString = "z24" }, + new() { FirstString = "1", SecondString = "z2" }, + new() { FirstString = "1", SecondString = "abcde" }, + new() { FirstString = "1", SecondString = "z15" }, + new() { FirstString = "1", SecondString = "abd" }, + new() { FirstString = "1", SecondString = "z1" }, + new() { FirstString = "1", SecondString = "z3" }, + new() { FirstString = "1", SecondString = "abc" }, + new() { FirstString = "1", SecondString = "z20" }, + new() { FirstString = "1", SecondString = "z5" }, + new() { FirstString = "1", SecondString = "abcd" }, + new() { FirstString = "1", SecondString = "z11" }, + new() { FirstString = "1", SecondString = "z22" }, + ]; + + TestRecord[] expected = + [ + new() { FirstString = "1", SecondString = "abc" }, + new() { FirstString = "1", SecondString = "abcd" }, + new() { FirstString = "1", SecondString = "abcde" }, + new() { FirstString = "1", SecondString = "abd" }, + new() { FirstString = "1", SecondString = "z1" }, + new() { FirstString = "1", SecondString = "z2" }, + new() { FirstString = "1", SecondString = "z3" }, + new() { FirstString = "1", SecondString = "z5" }, + new() { FirstString = "1", SecondString = "z11" }, + new() { FirstString = "1", SecondString = "z15" }, + new() { FirstString = "1", SecondString = "z20" }, + new() { FirstString = "1", SecondString = "z22" }, + new() { FirstString = "1", SecondString = "z24" }, + ]; + + var orderedRecords = testRecords + .NaturalOrderBy(r => r.FirstString) + .NaturalThenBy(r => r.SecondString); + + Assert.Equal(expected, orderedRecords); } [Fact] - public async Task ForEachAsync_FailingEither() + public void PreservesOrderingOfOrderBy() { - Either> results = - await new List {1, -1, 2} - .ForEachAsync(async value => - { - if (value == -1) - { - return await GetFailingEither(); - } - - return await GetSuccessfulEither(value); - }); - - Assert.True(results.IsLeft); + TestRecord[] testRecords = + [ + new() { FirstString = "2", SecondString = "a" }, + new() { FirstString = "2", SecondString = "b" }, + new() { FirstString = "1", SecondString = "b" }, + new() { FirstString = "1", SecondString = "a" }, + new() { FirstString = "3", SecondString = "b" }, + new() { FirstString = "3", SecondString = "a" }, + ]; + + TestRecord[] expected = + [ + new() { FirstString = "1", SecondString = "a" }, + new() { FirstString = "1", SecondString = "b" }, + new() { FirstString = "2", SecondString = "a" }, + new() { FirstString = "2", SecondString = "b" }, + new() { FirstString = "3", SecondString = "a" }, + new() { FirstString = "3", SecondString = "b" }, + ]; + + var orderedRecords = testRecords + .NaturalOrderBy(r => r.FirstString) + .NaturalThenBy(r => r.SecondString); + + Assert.Equal(expected, orderedRecords); } - [Fact] - public void SelectNullSafe_NotNull() + private record TestRecord { - var list = new List {1, 2, 3}; - var results = list.SelectNullSafe(value => value * 2).ToList(); - Assert.Equal(new List {2, 4, 6}, results); + public required string FirstString { get; init; } + public required string SecondString { get; init; } } + } - [Fact] - public void SelectNullSafe_Null() + [Fact] + public void ToDictionaryIndexed() + { + var source = new List { - var results = ((List?) null).SelectNullSafe(value => value * 2).ToList(); - Assert.Equal(new List(), results); - } + "a", + "b", + "c" + }; - [Fact] - public void IsNullOrEmpty_Null() - { - Assert.True(((List?) null).IsNullOrEmpty()); - } + var result = source.ToDictionaryIndexed(value => value, (value, index) => (Value: value, Index: index)); - [Fact] - public void IsNullOrEmpty_Empty() + var expected = new Dictionary { - Assert.True(new List().IsNullOrEmpty()); - } + {"a", ("a", 0)}, + {"b", ("b", 1)}, + {"c", ("c", 2)} + }; - [Fact] - public void IsNullOrEmpty_NeitherNullNorEmpty() - { - var list = new List {"foo", "bar"}; - Assert.False(list.IsNullOrEmpty()); - } + Assert.Equal(expected, result); + } - [Fact] - public void JoinToString() + [Fact] + public void DistinctByProperty() + { + var list = new List { - var list = new List {"foo", "bar", "baz"}; + new(1), + new(1), + new(2), + }; - Assert.Equal("foo-bar-baz", list.JoinToString('-')); - Assert.Equal("foo - bar - baz", list.JoinToString(" - ")); - Assert.Equal("foo, bar, baz", list.JoinToString(", ")); - } - - - [Fact] - public void Generate_Tuple2() - { - var (item1, item2) = new[] { "test1", "test2" }.ToTuple2(); - Assert.Equal("test1", item1); - Assert.Equal("test2", item2); - } - - [Fact] - public void Generate_Tuple2_LengthTooShort() - { - Assert.Throws(() => new[] { "test1" }.ToTuple2()); - } - - [Fact] - public void Generate_Tuple2_LengthTooLong() - { - Assert.Throws(() => new[] { "test1", "test2", "test3" }.ToTuple2()); - } - - [Fact] - public void Generate_Tuple3() - { - var (item1, item2, item3) = new[] { "test1", "test2", "test3" }.ToTuple3(); - Assert.Equal("test1", item1); - Assert.Equal("test2", item2); - Assert.Equal("test3", item3); - } - - [Fact] - public void Generate_Tuple3_LengthTooShort() - { - Assert.Throws(() => new[] { "test1", "test2" }.ToTuple3()); - } - - [Fact] - public void Generate_Tuple3_LengthTooLong() - { - Assert.Throws(() => new[] { "test1", "test2", "test3", "test4" }.ToTuple3()); - } + var distinct = list.DistinctByProperty(x => x.Value).ToList(); - [Fact] - public void ContainsAll_BothEmptyLists_ReturnsTrue() - { - var source = Enumerable.Empty(); - var values = Enumerable.Empty(); + Assert.Equal(2, distinct.Count); + Assert.Equal(1, distinct[0].Value); + Assert.Equal(2, distinct[1].Value); + } - bool containsAll = source.ContainsAll(values); + [Fact] + public void IndexOfFirst() + { + var result = new List { 1, 2, 3 } + .IndexOfFirst(value => value == 2); - Assert.True(containsAll); - } + Assert.Equal(1, result); + } - [Fact] - public void ContainsAll_SourceListIsEmpty_ReturnsFalse() - { - var source = Enumerable.Empty(); - var values = new List() { "" }; + [Fact] + public void IndexOfFirst_NoMatch() + { + var result = new List { 1, 2, 3 } + .IndexOfFirst(value => value == 4); - bool containsAll = source.ContainsAll(values); + Assert.Equal(-1, result); + } - Assert.False(containsAll); - } + [Fact] + public async Task ForEachAsync_SuccessfulEitherList() + { + Either> results = + await new List { 1, 2 } + .ForEachAsync(async value => + await GetSuccessfulEither(value)); - [Fact] - public void ContainsAll_ValuesListIsEmpty_ReturnsTrue() - { - var source = new List() { "" }; - var values = Enumerable.Empty(); + Assert.True(results.IsRight); - bool containsAll = source.ContainsAll(values); + var resultsList = results.Right; + Assert.Equal(2, resultsList.Count); + Assert.Contains(1, resultsList); + Assert.Contains(2, resultsList); + } - Assert.True(containsAll); - } + [Fact] + public async Task ForEachAsync_FailingEither() + { + Either> results = + await new List { 1, -1, 2 } + .ForEachAsync(async value => + { + if (value == -1) + { + return await GetFailingEither(); + } - [Fact] - public void ContainsAll_BothNullLists_ThrowsArgumentNullException() - { - IEnumerable source = null; - IEnumerable values = null; + return await GetSuccessfulEither(value); + }); - Assert.Throws(() => source.ContainsAll(values)); - } + Assert.True(results.IsLeft); + } - [Fact] - public void ContainsAll_SourceListIsNull_ThrowsArgumentNullException() - { - IEnumerable source = null; - var values = new List() { "" }; + [Fact] + public void SelectNullSafe_NotNull() + { + var list = new List { 1, 2, 3 }; + var results = list.SelectNullSafe(value => value * 2).ToList(); + Assert.Equal(new List { 2, 4, 6 }, results); + } - Assert.Throws(() => source.ContainsAll(values)); - } + [Fact] + public void SelectNullSafe_Null() + { + var results = ((List?)null).SelectNullSafe(value => value * 2).ToList(); + Assert.Equal(new List(), results); + } - [Fact] - public void ContainsAll_ValuesListIsNull_ThrowsArgumentNullException() - { - var source = new List() { "" }; - IEnumerable values = null; + [Fact] + public void IsNullOrEmpty_Null() + { + Assert.True(((List?)null).IsNullOrEmpty()); + } - Assert.Throws(() => source.ContainsAll(values)); - } + [Fact] + public void IsNullOrEmpty_Empty() + { + Assert.True(new List().IsNullOrEmpty()); + } - [Fact] - public void ContainsAll_SourceListDoesNotContainSingleValue_ReturnsFalse() - { - var source = new List() { "a", "b", "c" }; - var values = new List() { "d" }; + [Fact] + public void IsNullOrEmpty_NeitherNullNorEmpty() + { + var list = new List { "foo", "bar" }; + Assert.False(list.IsNullOrEmpty()); + } - bool containsAll = source.ContainsAll(values); + [Fact] + public void JoinToString() + { + var list = new List { "foo", "bar", "baz" }; - Assert.False(containsAll); - } + Assert.Equal("foo-bar-baz", list.JoinToString('-')); + Assert.Equal("foo - bar - baz", list.JoinToString(" - ")); + Assert.Equal("foo, bar, baz", list.JoinToString(", ")); + } - [Fact] - public void ContainsAll_SourceListContainsOneValueButNotAnother_ReturnsFalse() - { - var source = new List() { "a", "b", "c" }; - var values = new List() { "a", "d" }; - bool containsAll = source.ContainsAll(values); + [Fact] + public void Generate_Tuple2() + { + var (item1, item2) = new[] { "test1", "test2" }.ToTuple2(); + Assert.Equal("test1", item1); + Assert.Equal("test2", item2); + } - Assert.False(containsAll); - } + [Fact] + public void Generate_Tuple2_LengthTooShort() + { + Assert.Throws(() => new[] { "test1" }.ToTuple2()); + } - [Fact] - public void ContainsAll_SourceListContainsAllValues_ReturnsTrue() - { - var source = new List() { "a", "b", "c" }; - var values = new List() { "a", "b" }; + [Fact] + public void Generate_Tuple2_LengthTooLong() + { + Assert.Throws(() => new[] { "test1", "test2", "test3" }.ToTuple2()); + } + + [Fact] + public void Generate_Tuple3() + { + var (item1, item2, item3) = new[] { "test1", "test2", "test3" }.ToTuple3(); + Assert.Equal("test1", item1); + Assert.Equal("test2", item2); + Assert.Equal("test3", item3); + } - bool containsAll = source.ContainsAll(values); + [Fact] + public void Generate_Tuple3_LengthTooShort() + { + Assert.Throws(() => new[] { "test1", "test2" }.ToTuple3()); + } - Assert.True(containsAll); - } + [Fact] + public void Generate_Tuple3_LengthTooLong() + { + Assert.Throws(() => new[] { "test1", "test2", "test3", "test4" }.ToTuple3()); + } - private static async Task> GetSuccessfulEither(int value) - { - await Task.Delay(5); - return value; - } + [Fact] + public void ContainsAll_BothEmptyLists_ReturnsTrue() + { + var source = Enumerable.Empty(); + var values = Enumerable.Empty(); - private static async Task> GetFailingEither() - { - await Task.Delay(5); - return Unit.Instance; - } + bool containsAll = source.ContainsAll(values); - private class TestClass - { - public readonly int Value; + Assert.True(containsAll); + } + + [Fact] + public void ContainsAll_SourceListIsEmpty_ReturnsFalse() + { + var source = Enumerable.Empty(); + var values = new List() { "" }; + + bool containsAll = source.ContainsAll(values); + + Assert.False(containsAll); + } + + [Fact] + public void ContainsAll_ValuesListIsEmpty_ReturnsTrue() + { + var source = new List() { "" }; + var values = Enumerable.Empty(); + + bool containsAll = source.ContainsAll(values); + + Assert.True(containsAll); + } + + [Fact] + public void ContainsAll_BothNullLists_ThrowsArgumentNullException() + { + IEnumerable source = null; + IEnumerable values = null; + + Assert.Throws(() => source.ContainsAll(values)); + } + + [Fact] + public void ContainsAll_SourceListIsNull_ThrowsArgumentNullException() + { + IEnumerable source = null; + var values = new List() { "" }; + + Assert.Throws(() => source.ContainsAll(values)); + } + + [Fact] + public void ContainsAll_ValuesListIsNull_ThrowsArgumentNullException() + { + var source = new List() { "" }; + IEnumerable values = null; + + Assert.Throws(() => source.ContainsAll(values)); + } + + [Fact] + public void ContainsAll_SourceListDoesNotContainSingleValue_ReturnsFalse() + { + var source = new List() { "a", "b", "c" }; + var values = new List() { "d" }; + + bool containsAll = source.ContainsAll(values); + + Assert.False(containsAll); + } + + [Fact] + public void ContainsAll_SourceListContainsOneValueButNotAnother_ReturnsFalse() + { + var source = new List() { "a", "b", "c" }; + var values = new List() { "a", "d" }; + + bool containsAll = source.ContainsAll(values); + + Assert.False(containsAll); + } + + [Fact] + public void ContainsAll_SourceListContainsAllValues_ReturnsTrue() + { + var source = new List() { "a", "b", "c" }; + var values = new List() { "a", "b" }; + + bool containsAll = source.ContainsAll(values); + + Assert.True(containsAll); + } + + private static async Task> GetSuccessfulEither(int value) + { + await Task.Delay(5); + return value; + } + + private static async Task> GetFailingEither() + { + await Task.Delay(5); + return Unit.Instance; + } - public TestClass(int value) - { - Value = value; - } + private class TestClass + { + public readonly int Value; + + public TestClass(int value) + { + Value = value; } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs index 6549f9add2e..4a780e7f1d3 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/EnumerableExtensions.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Common.Utils; using GovUk.Education.ExploreEducationStatistics.Common.Model; +using NaturalSort.Extension; namespace GovUk.Education.ExploreEducationStatistics.Common.Extensions { @@ -296,5 +297,27 @@ public static bool ContainsAll(this IEnumerable source, IEnumerable val { return values.All(id => source.Contains(id)); } + + /// + /// Order some objects, according to a string key, in natural order for humans to read. + /// + public static IOrderedEnumerable NaturalOrderBy( + this IEnumerable source, + Func keySelector, + StringComparison comparison = StringComparison.OrdinalIgnoreCase) + { + return source.OrderBy(keySelector, comparison.WithNaturalSort()); + } + + /// + /// Subsequently order some objects, according to a string key, in natural order for humans to read. + /// + public static IOrderedEnumerable NaturalThenBy( + this IOrderedEnumerable source, + Func keySelector, + StringComparison comparison = StringComparison.OrdinalIgnoreCase) + { + return source.ThenBy(keySelector, comparison.WithNaturalSort()); + } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/FilterMappingPlanGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/FilterMappingPlanGeneratorExtensions.cs index bf128d63815..e7a8d7d5df1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/FilterMappingPlanGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/FilterMappingPlanGeneratorExtensions.cs @@ -17,7 +17,7 @@ public static Generator FilterMappingPlanFromFilterMeta( List? targetFilters = null) { var filterMappingPlanGenerator = fixture.Generator(); - + sourceFilters?.ForEach(sourceFilter => { var filterMappingGenerator = fixture @@ -26,7 +26,7 @@ public static Generator FilterMappingPlanFromFilterMeta( .DefaultMappableFilter() .WithLabel(sourceFilter.Label)) .WithPublicId(sourceFilter.PublicId); - + sourceFilter.Options.ForEach(option => { filterMappingGenerator.AddOptionMapping( @@ -43,13 +43,13 @@ public static Generator FilterMappingPlanFromFilterMeta( columnName: sourceFilter.PublicId, filterMappingGenerator); }); - + targetFilters?.ForEach(targetFilter => { var filterCandidateGenerator = fixture .DefaultFilterMappingCandidate() .WithLabel(targetFilter.Label); - + targetFilter.Options.ForEach(option => { filterCandidateGenerator.AddOptionCandidate( @@ -63,16 +63,16 @@ public static Generator FilterMappingPlanFromFilterMeta( columnName: targetFilter.PublicId, filterCandidateGenerator); }); - + return filterMappingPlanGenerator; } - + public static Generator AddFilterMapping( this Generator generator, string columnName, FilterMapping mapping) => generator.ForInstance(s => s.AddFilterMapping(columnName, mapping)); - + public static Generator AddFilterCandidate( this Generator generator, string columnName, @@ -175,7 +175,7 @@ public static Generator WithCandidateKey( this Generator generator, string candidateKey) => generator.ForInstance(s => s.SetCandidateKey(candidateKey)); - + public static Generator AddOptionMapping( this Generator generator, string sourceKey, @@ -214,7 +214,7 @@ public static InstanceSetters AddOptionMapping( string columnName, FilterOptionMapping mapping) => instanceSetter.Set((_, plan) => plan.OptionMappings.Add(columnName, mapping)); - + /** * FilterMappingCandidate */ @@ -228,7 +228,7 @@ public static Generator WithLabel( this Generator generator, string label) => generator.ForInstance(s => s.SetLabel(label)); - + public static Generator AddOptionCandidate( this Generator generator, string targetKey, @@ -304,7 +304,7 @@ public static Generator WithNoMapping( => generator.ForInstance(s => s .SetType(MappingType.None) .SetCandidateKey(null)); - + public static Generator WithAutoMapped( this Generator generator, string candidateKey) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/GeographicLevelMetaGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/GeographicLevelMetaGeneratorExtensions.cs index bf48ce9ca10..41d07c8fd9d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/GeographicLevelMetaGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/GeographicLevelMetaGeneratorExtensions.cs @@ -34,7 +34,7 @@ public static InstanceSetters SetDataSetVersion( DataSetVersion dataSetVersion) => setters .Set(m => m.DataSetVersion, dataSetVersion) - .Set(m => m.DataSetVersionId, dataSetVersion.Id); + .Set(m => m.DataSetVersionId, (_, m) => m.DataSetVersion.Id); public static InstanceSetters SetLevels( this InstanceSetters setters, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/LocationMappingPlanGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/LocationMappingPlanGeneratorExtensions.cs index ac2e2c4cea7..931fae17db7 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/LocationMappingPlanGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/LocationMappingPlanGeneratorExtensions.cs @@ -24,7 +24,7 @@ public static Generator LocationMappingPlanFromLocationMeta .Select(meta => meta.Level) .Distinct() .ToList(); - + levels.ForEach(level => { var levelGenerator = fixture @@ -32,7 +32,7 @@ public static Generator LocationMappingPlanFromLocationMeta var sourceLocationsForLevel = sourceLocations? .SingleOrDefault(meta => meta.Level == level); - + sourceLocationsForLevel?.Options.ForEach(option => { levelGenerator.AddMapping( @@ -43,10 +43,10 @@ public static Generator LocationMappingPlanFromLocationMeta .WithLabel(option.Label) .WithCodes(option.ToRow()))); }); - + var targetLocationsForLevel = targetLocations? .SingleOrDefault(meta => meta.Level == level); - + targetLocationsForLevel?.Options.ForEach(option => { levelGenerator.AddCandidate( @@ -56,7 +56,7 @@ public static Generator LocationMappingPlanFromLocationMeta .WithLabel(option.Label) .WithCodes(option.ToRow())); }); - + locationMappingPlanGenerator.AddLevel(level, levelGenerator); }); @@ -132,7 +132,7 @@ public static Generator WithCodes( urn: urn, laEstab: laEstab, ukprn: ukprn)); - + public static Generator WithCodes( this Generator generator, LocationOptionMetaRow metaRow) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Change.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Change.cs index 9b6d99913ac..6d49d62acb9 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Change.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Change.cs @@ -87,6 +87,18 @@ public class State public required int OptionId { get; set; } public required string PublicId { get; set; } + + public static State Create(FilterOptionMetaLink link) + { + return new State + { + Meta = link.Meta, + MetaId = link.MetaId, + Option = link.Option, + OptionId = link.OptionId, + PublicId = link.PublicId + }; + } } } @@ -161,6 +173,18 @@ public class State public required int OptionId { get; set; } public required string PublicId { get; set; } + + public static State Create(LocationOptionMetaLink link) + { + return new State + { + Meta = link.Meta, + MetaId = link.MetaId, + Option = link.Option, + OptionId = link.OptionId, + PublicId = link.PublicId + }; + } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionImportStage.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionImportStage.cs index d717d3b0476..e1051c88348 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionImportStage.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionImportStage.cs @@ -12,6 +12,7 @@ public enum DataSetVersionImportStage CreatingMappings, AutoMapping, ManualMapping, + CreatingChanges, ImportingData, WritingDataFiles, Completing diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/TimePeriodMeta.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/TimePeriodMeta.cs index 9376897bfbe..d48c2b29949 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/TimePeriodMeta.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/TimePeriodMeta.cs @@ -39,4 +39,19 @@ public void Configure(EntityTypeBuilder builder) .IsUnique(); } } + + private sealed class CodePeriodEqualityComparer : IEqualityComparer + { + public bool Equals(TimePeriodMeta? x, TimePeriodMeta? y) + { + return x?.Code == y?.Code && x?.Period == y?.Period; + } + + public int GetHashCode(TimePeriodMeta obj) + { + return HashCode.Combine((int)obj.Code, obj.Period); + } + } + + public static IEqualityComparer CodePeriodComparer { get; } = new CodePeriodEqualityComparer(); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CopyCsvFilesFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CopyCsvFilesFunctionTests.cs index 8de212aa16e..0bf0f05b33b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CopyCsvFilesFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CopyCsvFilesFunctionTests.cs @@ -47,7 +47,7 @@ await AddTestData(context => context.ReleaseFiles.AddRange(releaseDataFile, releaseMetaFile); }); - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage(), + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage(), releaseFileId: releaseDataFile.Id); var blobStorageService = GetRequiredService(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/HandleProcessingFailureFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/HandleProcessingFailureFunctionTests.cs index 054443dab17..53f610fe6f5 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/HandleProcessingFailureFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/HandleProcessingFailureFunctionTests.cs @@ -18,7 +18,7 @@ public async Task Success() // The stage which the failure occured in - This should not be altered by the handler const DataSetVersionImportStage failedStage = DataSetVersionImportStage.CopyingCsvFiles; - var (_, instanceId) = await CreateDataSet(failedStage); + var (_, instanceId) = await CreateDataSetInitialVersion(failedStage); var function = GetRequiredService(); await function.HandleProcessingFailure(instanceId, CancellationToken.None); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportDataFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportDataFunctionTests.cs index 8d79fb9c3fd..02a7112ee9c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportDataFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportDataFunctionTests.cs @@ -33,7 +33,7 @@ public class ImportDataTests( [MemberData(nameof(Data))] public async Task Success(ProcessorTestData testData) { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await ImportData(testData, dataSetVersion, instanceId); @@ -58,7 +58,7 @@ public async Task Success(ProcessorTestData testData) [MemberData(nameof(Data))] public async Task DuckDbDataTable_CorrectRowCount(ProcessorTestData testData) { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await ImportData(testData, dataSetVersion, instanceId); @@ -78,7 +78,7 @@ SELECT COUNT(*) [MemberData(nameof(Data))] public async Task DuckDbDataTable_CorrectColumns(ProcessorTestData testData) { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await ImportData(testData, dataSetVersion, instanceId); @@ -107,7 +107,7 @@ public async Task DuckDbDataTable_CorrectColumns(ProcessorTestData testData) [MemberData(nameof(Data))] public async Task DuckDbDataTable_CorrectDistinctGeographicLevels(ProcessorTestData testData) { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await ImportData(testData, dataSetVersion, instanceId); @@ -132,7 +132,7 @@ public async Task DuckDbDataTable_CorrectDistinctGeographicLevels(ProcessorTestD [MemberData(nameof(Data))] public async Task DuckDbDataTable_CorrectDistinctLocationOptions(ProcessorTestData testData) { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await ImportData(testData, dataSetVersion, instanceId); @@ -162,7 +162,7 @@ await Assert.AllAsync(testData.ExpectedLocations, [MemberData(nameof(Data))] public async Task DuckDbDataTable_CorrectDistinctFilterOptions(ProcessorTestData testData) { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await ImportData(testData, dataSetVersion, instanceId); @@ -197,7 +197,7 @@ await Assert.AllAsync(testData.ExpectedFilters, [MemberData(nameof(Data))] public async Task DuckDbDataTable_CorrectDistinctTimePeriods(ProcessorTestData testData) { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await ImportData(testData, dataSetVersion, instanceId); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportMetadataFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportMetadataFunctionTests.cs index d3b7b45492d..8bcc8af2919 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportMetadataFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportMetadataFunctionTests.cs @@ -46,7 +46,7 @@ public class ImportMetadataTests(ProcessorFunctionsIntegrationTestFixture fixtur [MemberData(nameof(TestDataFiles))] public async Task Success(ProcessorTestData testData) { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await ImportMetadata(testData, dataSetVersion, instanceId); @@ -99,7 +99,7 @@ public async Task Success(ProcessorTestData testData) [MemberData(nameof(TestDataFiles))] public async Task DataSetVersionMeta_CorrectGeographicLevels(ProcessorTestData testData) { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await ImportMetadata(testData, dataSetVersion, instanceId); @@ -119,7 +119,7 @@ public async Task DataSetVersionMeta_CorrectGeographicLevels(ProcessorTestData t [MemberData(nameof(TestDataFilesWithMetaInsertBatchSize))] public async Task DataSetVersionMeta_CorrectLocationOptions(ProcessorTestData testData, int metaInsertBatchSize) { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await ImportMetadata(testData, dataSetVersion, instanceId, metaInsertBatchSize); @@ -176,12 +176,10 @@ public async Task DataSetVersionMeta_CorrectLocationOptions_WithMappings( ProcessorTestData testData, int metaInsertBatchSize) { - var (sourceDataSetVersion, _) = await CreateDataSet(Stage.PreviousStage()); - - var (targetDataSetVersion, instanceId) = await CreateDataSetVersionAndImport( - dataSetId: sourceDataSetVersion.DataSet.Id, - importStage: DataSetVersionImportStage.ManualMapping, - versionMinor: 1); + var (sourceDataSetVersion, targetDataSetVersion, instanceId) = + await CreateDataSetInitialAndNextVersion( + nextVersionImportStage: DataSetVersionImportStage.ManualMapping, + nextVersionStatus: DataSetVersionStatus.Mapping); // In this test, we will create mappings for all the original location options. // 2 of these mappings will have candidates, and the rest will have no candidates @@ -294,7 +292,7 @@ await AddTestData(context => [MemberData(nameof(TestDataFiles))] public async Task DataSetVersionMeta_CorrectTimePeriods(ProcessorTestData testData) { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await ImportMetadata(testData, dataSetVersion, instanceId); @@ -323,7 +321,7 @@ public async Task DataSetVersionMeta_CorrectTimePeriods(ProcessorTestData testDa [MemberData(nameof(TestDataFilesWithMetaInsertBatchSize))] public async Task DataSetVersionMeta_CorrectFilters(ProcessorTestData testData, int metaInsertBatchSize) { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await ImportMetadata(testData, dataSetVersion, instanceId, metaInsertBatchSize); @@ -377,12 +375,10 @@ public async Task DataSetVersionMeta_CorrectFilters(ProcessorTestData testData, public async Task DataSetVersionMeta_CorrectFilters_WithMappings(ProcessorTestData testData, int metaInsertBatchSize) { - var (sourceDataSetVersion, _) = await CreateDataSet(Stage.PreviousStage()); - - var (targetDataSetVersion, instanceId) = await CreateDataSetVersionAndImport( - dataSetId: sourceDataSetVersion.DataSet.Id, - importStage: DataSetVersionImportStage.ManualMapping, - versionMinor: 1); + var (sourceDataSetVersion, targetDataSetVersion, instanceId) = + await CreateDataSetInitialAndNextVersion( + nextVersionImportStage: DataSetVersionImportStage.ManualMapping, + nextVersionStatus: DataSetVersionStatus.Mapping); // In this test, we will create mappings for all the original filter options. // 2 of these mappings will have candidates, and the rest will have no candidates @@ -526,7 +522,7 @@ await AddTestData(context => [MemberData(nameof(TestDataFiles))] public async Task DataSetVersionMeta_CorrectIndicators(ProcessorTestData testData) { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await ImportMetadata(testData, dataSetVersion, instanceId); @@ -554,7 +550,7 @@ public async Task DataSetVersionMeta_CorrectIndicators(ProcessorTestData testDat [MemberData(nameof(TestDataFiles))] public async Task DuckDbMeta_CorrectLocationOptions(ProcessorTestData testData) { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await ImportMetadata(testData, dataSetVersion, instanceId); @@ -589,7 +585,7 @@ public async Task DuckDbMeta_CorrectLocationOptions(ProcessorTestData testData) [MemberData(nameof(TestDataFiles))] public async Task DuckDbMeta_CorrectTimePeriods(ProcessorTestData testData) { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await ImportMetadata(testData, dataSetVersion, instanceId); @@ -618,7 +614,7 @@ public async Task DuckDbMeta_CorrectTimePeriods(ProcessorTestData testData) [MemberData(nameof(TestDataFiles))] public async Task DuckDbMeta_CorrectFilterOptions(ProcessorTestData testData) { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await ImportMetadata(testData, dataSetVersion, instanceId); @@ -663,7 +659,7 @@ public async Task DuckDbMeta_CorrectFilterOptions(ProcessorTestData testData) [MemberData(nameof(TestDataFiles))] public async Task DuckDbMeta_CorrectIndicators(ProcessorTestData testData) { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await ImportMetadata(testData, dataSetVersion, instanceId); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs index f7d28532146..382b84d6d4b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs @@ -1,15 +1,21 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Model; +using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Utils; using Microsoft.DurableTask; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging.Abstractions; using Moq; using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; +using FilterMeta = GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; @@ -45,6 +51,7 @@ public async Task Success() [ ActivityNames.UpdateFileStoragePath, ActivityNames.ImportMetadata, + ActivityNames.CreateChanges, ActivityNames.ImportData, ActivityNames.WriteDataFiles, ActivityNames.CompleteNextDataSetVersionImportProcessing @@ -117,6 +124,2964 @@ private static Mock DefaultMockOrchestrationContext(Gu } } + public abstract class CreateChangesTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : ProcessCompletionOfNextDataSetVersionImportFunctionTests(fixture) + { + protected const DataSetVersionImportStage Stage = DataSetVersionImportStage.CreatingChanges; + + protected async Task CreateChanges(Guid instanceId) + { + var function = GetRequiredService(); + await function.CreateChanges(instanceId, CancellationToken.None); + } + } + + public class CreateChangesFilterTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : CreateChangesTests(fixture) + { + [Fact] + public async Task FiltersAdded_ChangesContainOnlyAddedFilters() + { + var oldFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, s => + s.SetPublicId(SqidEncoder.Encode(1)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(1))) + .Generate(1))) + .GenerateList(1); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldFilterMeta); + + var newFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, UnchangedFilterMetaSetter(oldFilterMeta[0])) // Filter and ALL options unchanged + .ForIndex(1, s => + s.SetPublicId(SqidEncoder.Encode(2)) // Filter added + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(2))) // Filter Option added + .Generate(1))) + .ForIndex(2, s => + s.SetPublicId(SqidEncoder.Encode(3)) // Filter added + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) // Filter Option added + .Generate(1))) + .GenerateList(3); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + filterMetas: newFilterMeta); + + await CreateChanges(instanceId); + + var filterMetaChanges = await GetFilterMetaChanges(newVersion); + var filterOptionMetaChanges = await GetFilterOptionMetaChanges(newVersion); + + // 2 Filter additions + Assert.Equal(2, filterMetaChanges.Count); + Assert.All(filterMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + // No Filter Option additions + Assert.Empty(filterOptionMetaChanges); + + var newFilterMetas = newVersion.FilterMetas + .ToDictionary(m => m.PublicId); + + // Filter added + AssertSingleFilterAdded(filterMetaChanges, newFilterMetas[SqidEncoder.Encode(2)]); + + // Filter added + AssertSingleFilterAdded(filterMetaChanges, newFilterMetas[SqidEncoder.Encode(3)]); + } + + [Fact] + public async Task FiltersDeleted_ChangesContainOnlyDeletedFilters() + { + var oldFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, s => + s.SetPublicId(SqidEncoder.Encode(1)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(1))) + .Generate(1))) + .ForIndex(1, s => + s.SetPublicId(SqidEncoder.Encode(2)) // Filter and ALL options deleted + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(2))) + .Generate(1))) + .ForIndex(2, s => + s.SetPublicId(SqidEncoder.Encode(3)) // Filter and ALL options deleted + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) + .Generate(1))) + .GenerateList(3); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldFilterMeta); + + var newFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, UnchangedFilterMetaSetter(oldFilterMeta[0])) // Filter and ALL options unchanged + .GenerateList(1); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + filterMetas: newFilterMeta); + + await CreateChanges(instanceId); + + var filterMetaChanges = await GetFilterMetaChanges(newVersion); + var filterOptionMetaChanges = await GetFilterOptionMetaChanges(newVersion); + + // 2 Filter deletions + Assert.Equal(2, filterMetaChanges.Count); + Assert.All(filterMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + // No Filter Option deletions + Assert.Empty(filterOptionMetaChanges); + + var oldFilterMetas = originalVersion.FilterMetas + .ToDictionary(m => m.PublicId); + + // Filter deleted + AssertSingleFilterDeleted(filterMetaChanges, oldFilterMetas[SqidEncoder.Encode(2)]); + + // Filter deleted + AssertSingleFilterDeleted(filterMetaChanges, oldFilterMetas[SqidEncoder.Encode(3)]); + } + + [Fact] + public async Task FiltersUpdatedOptionsAdded_ChangesContainOnlyUpdatedFiltersAndAddedOptions() + { + var oldFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, s => + s.SetPublicId(SqidEncoder.Encode(1)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(1))) + .Generate(1))) + .ForIndex(1, s => + s.SetPublicId(SqidEncoder.Encode(2)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(2))) + .Generate(1))) + .ForIndex(2, s => + s.SetPublicId(SqidEncoder.Encode(3)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) + .Generate(1))) + .GenerateList(3); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldFilterMeta); + + var newFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, UnchangedFilterMetaSetter(oldFilterMeta[0])) // Filter and ALL options unchanged + .ForIndex(1, s => + s.SetPublicId(SqidEncoder.Encode(2)) // Filter updated + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, + UnchangedFilterOptionMetaLinkSetter(oldFilterMeta[1] + .OptionLinks[0])) // Filter Option unchanged + .ForIndex(1, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(4))) // Filter Option added + .ForIndex(2, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(5))) // Filter Option added + .Generate(3))) + .ForIndex(2, s => + s.SetPublicId(SqidEncoder.Encode(3)) // Filter updated + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, + UnchangedFilterOptionMetaLinkSetter(oldFilterMeta[2] + .OptionLinks[0])) // Filter Option unchanged + .ForIndex(1, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(6))) // Filter Option added + .ForIndex(2, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(7))) // Filter Option added + .Generate(3))) + .GenerateList(3); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + filterMetas: newFilterMeta); + + await CreateChanges(instanceId); + + var filterMetaChanges = await GetFilterMetaChanges(newVersion); + var filterOptionMetaChanges = await GetFilterOptionMetaChanges(newVersion); + + // 2 Filter changes + Assert.Equal(2, filterMetaChanges.Count); + Assert.All(filterMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + // 4 Filter Option changes + Assert.Equal(4, filterOptionMetaChanges.Count); + Assert.All(filterOptionMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + var oldFilterMetas = originalVersion.FilterMetas + .ToDictionary(m => m.PublicId); + + var newFilterMetas = newVersion.FilterMetas + .ToDictionary( + m => m.PublicId, + m => new { Meta = m, OptionLinks = m.OptionLinks.ToDictionary(l => l.PublicId) }); + + // Filter updated + AssertSingleFilterUpdated( + changes: filterMetaChanges, + expectedOldFilter: oldFilterMetas[SqidEncoder.Encode(2)], + expectedNewFilter: newFilterMetas[SqidEncoder.Encode(2)].Meta); + + // Filter updated + AssertSingleFilterUpdated( + changes: filterMetaChanges, + expectedOldFilter: oldFilterMetas[SqidEncoder.Encode(3)], + expectedNewFilter: newFilterMetas[SqidEncoder.Encode(3)].Meta); + + // Filter Option added + AssertSingleFilterOptionAdded(filterOptionMetaChanges, + newFilterMetas[SqidEncoder.Encode(2)].OptionLinks[SqidEncoder.Encode(4)]); + + // Filter Option added + AssertSingleFilterOptionAdded(filterOptionMetaChanges, + newFilterMetas[SqidEncoder.Encode(2)].OptionLinks[SqidEncoder.Encode(5)]); + + // Filter Option added + AssertSingleFilterOptionAdded(filterOptionMetaChanges, + newFilterMetas[SqidEncoder.Encode(3)].OptionLinks[SqidEncoder.Encode(6)]); + + // Filter Option added + AssertSingleFilterOptionAdded(filterOptionMetaChanges, + newFilterMetas[SqidEncoder.Encode(3)].OptionLinks[SqidEncoder.Encode(7)]); + } + + [Fact] + public async Task FiltersUpdatedOptionsDeleted_ChangesContainOnlyUpdatedFiltersAndDeletedOptions() + { + var oldFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, s => + s.SetPublicId(SqidEncoder.Encode(1)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(1))) + .Generate(1))) + .ForIndex(1, s => + s.SetPublicId(SqidEncoder.Encode(2)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(2))) + .ForIndex(1, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) // Filter Option deleted + .ForIndex(2, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(4))) // Filter Option deleted + .Generate(3))) + .ForIndex(2, s => + s.SetPublicId(SqidEncoder.Encode(3)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(5))) + .ForIndex(1, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(6))) // Filter Option deleted + .ForIndex(2, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(7))) // Filter Option deleted + .Generate(3))) + .GenerateList(3); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldFilterMeta); + + var newFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, UnchangedFilterMetaSetter(oldFilterMeta[0])) // Filter and ALL options unchanged + .ForIndex(1, s => + s.SetPublicId(SqidEncoder.Encode(2)) // Filter updated + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, + UnchangedFilterOptionMetaLinkSetter(oldFilterMeta[1] + .OptionLinks[0])) // Filter Option unchanged + .Generate(1))) + .ForIndex(2, s => + s.SetPublicId(SqidEncoder.Encode(3)) // Filter updated + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, + UnchangedFilterOptionMetaLinkSetter(oldFilterMeta[2] + .OptionLinks[0])) // Filter Option unchanged + .Generate(1))) + .GenerateList(3); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + filterMetas: newFilterMeta); + + await CreateChanges(instanceId); + + var filterMetaChanges = await GetFilterMetaChanges(newVersion); + var filterOptionMetaChanges = await GetFilterOptionMetaChanges(newVersion); + + // 2 Filter changes + Assert.Equal(2, filterMetaChanges.Count); + Assert.All(filterMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + // 4 Filter Option changes + Assert.Equal(4, filterOptionMetaChanges.Count); + Assert.All(filterOptionMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + var oldFilterMetas = originalVersion.FilterMetas + .ToDictionary( + m => m.PublicId, + m => new { Meta = m, OptionLinks = m.OptionLinks.ToDictionary(l => l.PublicId) }); + + var newFilterMetas = newVersion.FilterMetas + .ToDictionary(m => m.PublicId); + + // Filter updated + AssertSingleFilterUpdated( + changes: filterMetaChanges, + expectedOldFilter: oldFilterMetas[SqidEncoder.Encode(2)].Meta, + expectedNewFilter: newFilterMetas[SqidEncoder.Encode(2)]); + + // Filter updated + AssertSingleFilterUpdated( + changes: filterMetaChanges, + expectedOldFilter: oldFilterMetas[SqidEncoder.Encode(3)].Meta, + expectedNewFilter: newFilterMetas[SqidEncoder.Encode(3)]); + + // Filter Option deleted + AssertSingleFilterOptionDeleted(filterOptionMetaChanges, + oldFilterMetas[SqidEncoder.Encode(2)].OptionLinks[SqidEncoder.Encode(3)]); + + // Filter Option deleted + AssertSingleFilterOptionDeleted(filterOptionMetaChanges, + oldFilterMetas[SqidEncoder.Encode(2)].OptionLinks[SqidEncoder.Encode(4)]); + + // Filter Option deleted + AssertSingleFilterOptionDeleted(filterOptionMetaChanges, + oldFilterMetas[SqidEncoder.Encode(3)].OptionLinks[SqidEncoder.Encode(6)]); + + // Filter Option deleted + AssertSingleFilterOptionDeleted(filterOptionMetaChanges, + oldFilterMetas[SqidEncoder.Encode(3)].OptionLinks[SqidEncoder.Encode(7)]); + } + + [Fact] + public async Task FiltersUpdatedOptionsUpdated_ChangesContainOnlyUpdatedFiltersAndUpdatedOptions() + { + var oldFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, s => + s.SetPublicId(SqidEncoder.Encode(1)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(1))) + .Generate(1))) + .ForIndex(1, s => + s.SetPublicId(SqidEncoder.Encode(2)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(2))) + .ForIndex(1, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) + .ForIndex(2, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(4))) + .Generate(3))) + .ForIndex(2, s => + s.SetPublicId(SqidEncoder.Encode(3)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(5))) + .ForIndex(1, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(6))) + .ForIndex(2, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(7))) + .Generate(3))) + .GenerateList(3); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldFilterMeta); + + var newFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, UnchangedFilterMetaSetter(oldFilterMeta[0])) // Filter and ALL options unchanged + .ForIndex(1, s => + s.SetPublicId(SqidEncoder.Encode(2)) // Filter updated + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, + UnchangedFilterOptionMetaLinkSetter(oldFilterMeta[1] + .OptionLinks[0])) // Filter Option unchanged + .ForIndex(1, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) // Filter Option updated + .ForIndex(2, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(4))) // Filter Option updated + .Generate(3))) + .ForIndex(2, s => + s.SetPublicId(SqidEncoder.Encode(3)) // Filter updated + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, + UnchangedFilterOptionMetaLinkSetter(oldFilterMeta[2] + .OptionLinks[0])) // Filter Option unchanged + .ForIndex(1, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(6))) // Filter Option updated + .ForIndex(2, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(7))) // Filter Option updated + .Generate(3))) + .GenerateList(3); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + filterMetas: newFilterMeta); + + await CreateChanges(instanceId); + + var filterMetaChanges = await GetFilterMetaChanges(newVersion); + var filterOptionMetaChanges = await GetFilterOptionMetaChanges(newVersion); + + // 2 Filter changes + Assert.Equal(2, filterMetaChanges.Count); + Assert.All(filterMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + // 4 Filter Option changes + Assert.Equal(4, filterOptionMetaChanges.Count); + Assert.All(filterOptionMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + var oldFilterMetas = originalVersion.FilterMetas + .ToDictionary( + m => m.PublicId, + m => new { Meta = m, OptionLinks = m.OptionLinks.ToDictionary(l => l.PublicId) }); + + var newFilterMetas = newVersion.FilterMetas + .ToDictionary( + m => m.PublicId, + m => new { Meta = m, OptionLinks = m.OptionLinks.ToDictionary(l => l.PublicId) }); + + // Filter updated + AssertSingleFilterUpdated( + changes: filterMetaChanges, + expectedOldFilter: oldFilterMetas[SqidEncoder.Encode(2)].Meta, + expectedNewFilter: newFilterMetas[SqidEncoder.Encode(2)].Meta); + + // Filter updated + AssertSingleFilterUpdated( + changes: filterMetaChanges, + expectedOldFilter: oldFilterMetas[SqidEncoder.Encode(3)].Meta, + expectedNewFilter: newFilterMetas[SqidEncoder.Encode(3)].Meta); + + // Filter Option updated + AssertSingleFilterOptionUpdated( + changes: filterOptionMetaChanges, + expectedOldOptionLink: oldFilterMetas[SqidEncoder.Encode(2)] + .OptionLinks[SqidEncoder.Encode(3)], + expectedNewOptionLink: newFilterMetas[SqidEncoder.Encode(2)] + .OptionLinks[SqidEncoder.Encode(3)]); + + // Filter Option updated + AssertSingleFilterOptionUpdated( + changes: filterOptionMetaChanges, + expectedOldOptionLink: oldFilterMetas[SqidEncoder.Encode(2)] + .OptionLinks[SqidEncoder.Encode(4)], + expectedNewOptionLink: newFilterMetas[SqidEncoder.Encode(2)] + .OptionLinks[SqidEncoder.Encode(4)]); + + // Filter Option updated + AssertSingleFilterOptionUpdated( + changes: filterOptionMetaChanges, + expectedOldOptionLink: oldFilterMetas[SqidEncoder.Encode(3)] + .OptionLinks[SqidEncoder.Encode(6)], + expectedNewOptionLink: newFilterMetas[SqidEncoder.Encode(3)] + .OptionLinks[SqidEncoder.Encode(6)]); + + // Filter Option updated + AssertSingleFilterOptionUpdated( + changes: filterOptionMetaChanges, + expectedOldOptionLink: oldFilterMetas[SqidEncoder.Encode(3)] + .OptionLinks[SqidEncoder.Encode(7)], + expectedNewOptionLink: newFilterMetas[SqidEncoder.Encode(3)] + .OptionLinks[SqidEncoder.Encode(7)]); + } + + [Fact] + public async Task FiltersUpdatedOptionsUnchanged_ChangesContainOnlyUpdatedFilters() + { + var oldFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, s => + s.SetPublicId(SqidEncoder.Encode(1)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(1))) + .Generate(1))) + .ForIndex(1, s => + s.SetPublicId(SqidEncoder.Encode(2)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(2))) + .ForIndex(1, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) + .Generate(2))) + .ForIndex(2, s => + s.SetPublicId(SqidEncoder.Encode(3)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(4))) + .ForIndex(1, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(5))) + .Generate(2))) + .GenerateList(3); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldFilterMeta); + + var newFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, UnchangedFilterMetaSetter(oldFilterMeta[0])) // Filter and ALL options unchanged + .ForIndex(1, s => + s.SetPublicId(SqidEncoder.Encode(2)) // Filter updated + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, + UnchangedFilterOptionMetaLinkSetter(oldFilterMeta[1] + .OptionLinks[0])) // Filter Option unchanged + .ForIndex(1, + UnchangedFilterOptionMetaLinkSetter(oldFilterMeta[1] + .OptionLinks[1])) // Filter Option unchanged + .Generate(2))) + .ForIndex(2, s => + s.SetPublicId(SqidEncoder.Encode(3)) // Filter updated + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, + UnchangedFilterOptionMetaLinkSetter(oldFilterMeta[2] + .OptionLinks[0])) // Filter Option unchanged + .ForIndex(1, + UnchangedFilterOptionMetaLinkSetter(oldFilterMeta[2] + .OptionLinks[1])) // Filter Option unchanged + .Generate(2))) + .GenerateList(3); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + filterMetas: newFilterMeta); + + await CreateChanges(instanceId); + + var filterMetaChanges = await GetFilterMetaChanges(newVersion); + var filterOptionMetaChanges = await GetFilterOptionMetaChanges(newVersion); + + // 2 Filter changes + Assert.Equal(2, filterMetaChanges.Count); + Assert.All(filterMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + // No Filter Option changes + Assert.Empty(filterOptionMetaChanges); + + var oldFilterMetas = originalVersion.FilterMetas + .ToDictionary(m => m.PublicId); + + var newFilterMetas = newVersion.FilterMetas + .ToDictionary(m => m.PublicId); + + // Filter updated + AssertSingleFilterUpdated( + changes: filterMetaChanges, + expectedOldFilter: oldFilterMetas[SqidEncoder.Encode(2)], + expectedNewFilter: newFilterMetas[SqidEncoder.Encode(2)]); + + // Filter updated + AssertSingleFilterUpdated( + changes: filterMetaChanges, + expectedOldFilter: oldFilterMetas[SqidEncoder.Encode(3)], + expectedNewFilter: newFilterMetas[SqidEncoder.Encode(3)]); + } + + [Fact] + public async Task FiltersUnchangedOptionsAdded_ChangesContainOnlyAddedOptions() + { + var oldFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, s => + s.SetPublicId(SqidEncoder.Encode(1)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(1))) + .Generate(1))) + .ForIndex(1, s => + s.SetPublicId(SqidEncoder.Encode(2)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(2))) + .Generate(1))) + .GenerateList(2); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldFilterMeta); + + var newFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, UnchangedFilterMetaSetter( + filterMeta: oldFilterMeta[0], // Filter unchanged + newOptionLinks: () => DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, + UnchangedFilterOptionMetaLinkSetter(oldFilterMeta[0] + .OptionLinks[0])) // Filter Option unchanged + .ForIndex(1, s => + s.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) // Filter Option added + .ForIndex(2, s => + s.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(4))) // Filter Option added + .Generate(3))) + .ForIndex(1, UnchangedFilterMetaSetter( + filterMeta: oldFilterMeta[1], // Filter unchanged + newOptionLinks: () => DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, + UnchangedFilterOptionMetaLinkSetter(oldFilterMeta[1] + .OptionLinks[0])) // Filter Option unchanged + .ForIndex(1, s => + s.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(5))) // Filter Option added + .ForIndex(2, s => + s.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(6))) // Filter Option added + .Generate(3))) + .GenerateList(2); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + filterMetas: newFilterMeta); + + await CreateChanges(instanceId); + + var filterMetaChanges = await GetFilterMetaChanges(newVersion); + var filterOptionMetaChanges = await GetFilterOptionMetaChanges(newVersion); + + // No Filter changes + Assert.Empty(filterMetaChanges); + + // 4 Filter Option changes + Assert.Equal(4, filterOptionMetaChanges.Count); + Assert.All(filterOptionMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + var newFilterMetas = newVersion.FilterMetas + .ToDictionary( + m => m.PublicId, + m => m.OptionLinks.ToDictionary(l => l.PublicId)); + + // Filter Option added + AssertSingleFilterOptionAdded(filterOptionMetaChanges, + newFilterMetas[SqidEncoder.Encode(1)][SqidEncoder.Encode(3)]); + + // Filter Option added + AssertSingleFilterOptionAdded(filterOptionMetaChanges, + newFilterMetas[SqidEncoder.Encode(1)][SqidEncoder.Encode(4)]); + + // Filter Option added + AssertSingleFilterOptionAdded(filterOptionMetaChanges, + newFilterMetas[SqidEncoder.Encode(2)][SqidEncoder.Encode(5)]); + + // Filter Option added + AssertSingleFilterOptionAdded(filterOptionMetaChanges, + newFilterMetas[SqidEncoder.Encode(2)][SqidEncoder.Encode(6)]); + } + + [Fact] + public async Task FiltersUnchangedOptionsDeleted_ChangesContainOnlyDeletedOptions() + { + var oldFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, s => + s.SetPublicId(SqidEncoder.Encode(1)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(1))) + .ForIndex(1, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(2))) // Filter Option deleted + .ForIndex(2, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) // Filter Option deleted + .Generate(3))) + .ForIndex(1, s => + s.SetPublicId(SqidEncoder.Encode(2)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(4))) + .ForIndex(1, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(5))) // Filter Option deleted + .ForIndex(2, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(6))) // Filter Option deleted + .Generate(3))) + .GenerateList(2); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldFilterMeta); + + var newFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, UnchangedFilterMetaSetter( + filterMeta: oldFilterMeta[0], // Filter unchanged + newOptionLinks: () => DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, + UnchangedFilterOptionMetaLinkSetter(oldFilterMeta[0] + .OptionLinks[0])) // Filter Option unchanged + .Generate(1))) + .ForIndex(1, UnchangedFilterMetaSetter( + filterMeta: oldFilterMeta[1], // Filter unchanged + newOptionLinks: () => DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, + UnchangedFilterOptionMetaLinkSetter(oldFilterMeta[1] + .OptionLinks[0])) // Filter Option unchanged + .Generate(1))) + .GenerateList(2); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + filterMetas: newFilterMeta); + + await CreateChanges(instanceId); + + var filterMetaChanges = await GetFilterMetaChanges(newVersion); + var filterOptionMetaChanges = await GetFilterOptionMetaChanges(newVersion); + + // No Filter changes + Assert.Empty(filterMetaChanges); + + // 4 Filter Option changes + Assert.Equal(4, filterOptionMetaChanges.Count); + Assert.All(filterOptionMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + var oldFilterMetas = originalVersion.FilterMetas + .ToDictionary( + m => m.PublicId, + m => m.OptionLinks.ToDictionary(l => l.PublicId)); + + // Filter Option deleted + AssertSingleFilterOptionDeleted(filterOptionMetaChanges, + oldFilterMetas[SqidEncoder.Encode(1)][SqidEncoder.Encode(2)]); + + // Filter Option deleted + AssertSingleFilterOptionDeleted(filterOptionMetaChanges, + oldFilterMetas[SqidEncoder.Encode(1)][SqidEncoder.Encode(3)]); + + // Filter Option deleted + AssertSingleFilterOptionDeleted(filterOptionMetaChanges, + oldFilterMetas[SqidEncoder.Encode(2)][SqidEncoder.Encode(5)]); + + // Filter Option deleted + AssertSingleFilterOptionDeleted(filterOptionMetaChanges, + oldFilterMetas[SqidEncoder.Encode(2)][SqidEncoder.Encode(6)]); + } + + [Fact] + public async Task FiltersUnchangedOptionsUpdated_ChangesContainOnlyUpdatedOptions() + { + var oldFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, s => + s.SetPublicId(SqidEncoder.Encode(1)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(1))) + .ForIndex(1, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(2))) + .ForIndex(2, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) + .Generate(3))) + .ForIndex(1, s => + s.SetPublicId(SqidEncoder.Encode(2)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(4))) + .ForIndex(1, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(5))) + .ForIndex(2, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(6))) + .Generate(3))) + .GenerateList(2); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldFilterMeta); + + var newFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, UnchangedFilterMetaSetter( + filterMeta: oldFilterMeta[0], // Filter unchanged + newOptionLinks: () => DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, + UnchangedFilterOptionMetaLinkSetter(oldFilterMeta[0] + .OptionLinks[0])) // Filter Option unchanged + .ForIndex(1, s => + s.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(2))) // Filter Option updated + .ForIndex(2, s => + s.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) // Filter Option updated + .Generate(3))) + .ForIndex(1, UnchangedFilterMetaSetter( + filterMeta: oldFilterMeta[1], // Filter unchanged + newOptionLinks: () => DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, + UnchangedFilterOptionMetaLinkSetter(oldFilterMeta[1] + .OptionLinks[0])) // Filter Option unchanged + .ForIndex(1, s => + s.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(5))) // Filter Option updated + .ForIndex(2, s => + s.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(6))) // Filter Option updated + .Generate(3))) + .GenerateList(2); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + filterMetas: newFilterMeta); + + await CreateChanges(instanceId); + + var filterMetaChanges = await GetFilterMetaChanges(newVersion); + var filterOptionMetaChanges = await GetFilterOptionMetaChanges(newVersion); + + // No Filter changes + Assert.Empty(filterMetaChanges); + + // 4 Filter Option changes + Assert.Equal(4, filterOptionMetaChanges.Count); + Assert.All(filterOptionMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + var oldFilterMetas = originalVersion.FilterMetas + .ToDictionary( + m => m.PublicId, + m => m.OptionLinks.ToDictionary(l => l.PublicId)); + + var newFilterMetas = newVersion.FilterMetas + .ToDictionary( + m => m.PublicId, + m => m.OptionLinks.ToDictionary(l => l.PublicId)); + + // Filter Option updated + AssertSingleFilterOptionUpdated( + changes: filterOptionMetaChanges, + expectedOldOptionLink: oldFilterMetas[SqidEncoder.Encode(1)][SqidEncoder.Encode(2)], + expectedNewOptionLink: newFilterMetas[SqidEncoder.Encode(1)][SqidEncoder.Encode(2)]); + + // Filter Option updated + AssertSingleFilterOptionUpdated( + changes: filterOptionMetaChanges, + expectedOldOptionLink: oldFilterMetas[SqidEncoder.Encode(1)][SqidEncoder.Encode(3)], + expectedNewOptionLink: newFilterMetas[SqidEncoder.Encode(1)][SqidEncoder.Encode(3)]); + + // Filter Option updated + AssertSingleFilterOptionUpdated( + changes: filterOptionMetaChanges, + expectedOldOptionLink: oldFilterMetas[SqidEncoder.Encode(2)][SqidEncoder.Encode(5)], + expectedNewOptionLink: newFilterMetas[SqidEncoder.Encode(2)][SqidEncoder.Encode(5)]); + + // Filter Option updated + AssertSingleFilterOptionUpdated( + changes: filterOptionMetaChanges, + expectedOldOptionLink: oldFilterMetas[SqidEncoder.Encode(2)][SqidEncoder.Encode(6)], + expectedNewOptionLink: newFilterMetas[SqidEncoder.Encode(2)][SqidEncoder.Encode(6)]); + } + + [Fact] + public async Task FiltersUnchangedOptionsUnchanged_ChangesAreEmpty() + { + var oldFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, s => + s.SetPublicId(SqidEncoder.Encode(1)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(1))) + .Generate(1))) + .ForIndex(1, s => + s.SetPublicId(SqidEncoder.Encode(2)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(2))) + .Generate(1))) + .GenerateList(2); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldFilterMeta); + + var newFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, UnchangedFilterMetaSetter(oldFilterMeta[0])) // Filter and ALL options unchanged + .ForIndex(1, UnchangedFilterMetaSetter(oldFilterMeta[1])) // Filter and ALL options unchanged + .GenerateList(2); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + filterMetas: newFilterMeta); + + await CreateChanges(instanceId); + + var filterMetaChanges = await GetFilterMetaChanges(newVersion); + var filterOptionMetaChanges = await GetFilterOptionMetaChanges(newVersion); + + // No Filter changes + Assert.Empty(filterMetaChanges); + + // No Filter Option changes + Assert.Empty(filterOptionMetaChanges); + } + + [Fact] + public async Task FiltersAddedAndDeletedAndUpdated_ChangesInsertedIntoDatabaseInCorrectOrder() + { + var oldFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, s => + s.SetPublicId(SqidEncoder.Encode(1)) // Filter deleted + .SetLabel("f")) + .ForIndex(1, s => + s.SetPublicId(SqidEncoder.Encode(2)) // Filter deleted + .SetLabel("a")) + .ForIndex(2, s => + s.SetPublicId(SqidEncoder.Encode(3)) + .SetLabel("e")) + .ForIndex(3, s => + s.SetPublicId(SqidEncoder.Encode(4)) + .SetLabel("b")) + .GenerateList(4); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldFilterMeta); + + var newFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, s => s.SetPublicId(SqidEncoder.Encode(3))) // Filter updated + .ForIndex(1, s => s.SetPublicId(SqidEncoder.Encode(4))) // Filter updated + .ForIndex(2, s => + s.SetPublicId(SqidEncoder.Encode(5)) // Filter added + .SetLabel("d")) + .ForIndex(3, s => + s.SetPublicId(SqidEncoder.Encode(6)) // Filter added + .SetLabel("c")) + .GenerateList(4); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + filterMetas: newFilterMeta); + + await CreateChanges(instanceId); + + var filterMetaChanges = await GetFilterMetaChanges(newVersion); + + // 6 Filter changes + Assert.Equal(6, filterMetaChanges.Count); + Assert.All(filterMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + var oldFilterMetas = originalVersion.FilterMetas + .ToDictionary(m => m.PublicId); + + var newFilterMetas = newVersion.FilterMetas + .ToDictionary(m => m.PublicId); + + // The changes should be inserted into each database table ordered alphabetically by 'Label'. + // They should also be ordered such that all deletions come first, updates next, and additions last. + + // Therefore, the expected order of Filter changes are (as per their Public IDs): + // Sqid 2 deleted + // Sqid 1 deleted + // Sqid 4 updated + // Sqid 3 updated + // Sqid 6 added + // Sqid 5 added + + AssertFilterDeleted( + expectedFilter: oldFilterMetas[SqidEncoder.Encode(2)], + change: filterMetaChanges[0]); + AssertFilterDeleted( + expectedFilter: oldFilterMetas[SqidEncoder.Encode(1)], + change: filterMetaChanges[1]); + AssertFilterUpdated( + expectedOldFilter: oldFilterMetas[SqidEncoder.Encode(4)], + expectedNewFilter: newFilterMetas[SqidEncoder.Encode(4)], + change: filterMetaChanges[2]); + AssertFilterUpdated( + expectedOldFilter: oldFilterMetas[SqidEncoder.Encode(3)], + expectedNewFilter: newFilterMetas[SqidEncoder.Encode(3)], + change: filterMetaChanges[3]); + AssertFilterAdded( + expectedFilter: newFilterMetas[SqidEncoder.Encode(6)], + change: filterMetaChanges[4]); + AssertFilterAdded( + expectedFilter: newFilterMetas[SqidEncoder.Encode(5)], + change: filterMetaChanges[5]); + } + + [Fact] + public async Task FiltersOptionsAddedAndDeletedAndUpdated_ChangesInsertedIntoDatabaseInCorrectOrder() + { + var oldFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, s => + s.SetPublicId(SqidEncoder.Encode(1)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption( + DataFixture.DefaultFilterOptionMeta() + .WithLabel("d")) + .SetPublicId(SqidEncoder.Encode(1))) // Filter Option deleted + .ForIndex(1, ls => + ls.SetOption( + DataFixture.DefaultFilterOptionMeta() + .WithLabel("a")) + .SetPublicId(SqidEncoder.Encode(2))) // Filter Option deleted + .ForIndex(2, ls => + ls.SetOption( + DataFixture.DefaultFilterOptionMeta() + .WithLabel("b")) + .SetPublicId(SqidEncoder.Encode(3))) + .ForIndex(3, ls => + ls.SetOption( + DataFixture.DefaultFilterOptionMeta() + .WithLabel("k")) + .SetPublicId(SqidEncoder.Encode(4))) + .Generate(4))) + .ForIndex(1, s => + s.SetPublicId(SqidEncoder.Encode(2)) + .SetOptionLinks(() => + DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption( + DataFixture.DefaultFilterOptionMeta() + .WithLabel("c")) + .SetPublicId(SqidEncoder.Encode(5))) // Filter Option deleted + .ForIndex(1, ls => + ls.SetOption( + DataFixture.DefaultFilterOptionMeta() + .WithLabel("h")) + .SetPublicId(SqidEncoder.Encode(6))) // Filter Option deleted + .ForIndex(2, ls => + ls.SetOption( + DataFixture.DefaultFilterOptionMeta() + .WithLabel("f")) + .SetPublicId(SqidEncoder.Encode(7))) + .ForIndex(3, ls => + ls.SetOption( + DataFixture.DefaultFilterOptionMeta() + .WithLabel("i")) + .SetPublicId(SqidEncoder.Encode(8))) + .Generate(4))) + .GenerateList(2); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldFilterMeta); + + var newFilterMeta = DataFixture.DefaultFilterMeta() + .ForIndex(0, UnchangedFilterMetaSetter( + filterMeta: oldFilterMeta[0], + newOptionLinks: () => DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, s => + s.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) // Filter Option updated + .ForIndex(1, s => + s.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(4))) // Filter Option updated + .ForIndex(2, s => + s.SetOption( + DataFixture.DefaultFilterOptionMeta() + .WithLabel("j")) + .SetPublicId(SqidEncoder.Encode(9))) // Filter Option added + .ForIndex(3, s => + s.SetOption( + DataFixture.DefaultFilterOptionMeta() + .WithLabel("e")) + .SetPublicId(SqidEncoder.Encode(10))) // Filter Option added + .Generate(4))) + .ForIndex(1, UnchangedFilterMetaSetter( + filterMeta: oldFilterMeta[1], + newOptionLinks: () => DataFixture.DefaultFilterOptionMetaLink() + .ForIndex(0, s => + s.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(7))) // Filter Option updated + .ForIndex(1, s => + s.SetOption(DataFixture.DefaultFilterOptionMeta()) + .SetPublicId(SqidEncoder.Encode(8))) // Filter Option updated + .ForIndex(2, s => + s.SetOption( + DataFixture.DefaultFilterOptionMeta() + .WithLabel("g")) + .SetPublicId(SqidEncoder.Encode(11))) // Filter Option added + .ForIndex(3, s => + s.SetOption( + DataFixture.DefaultFilterOptionMeta() + .WithLabel("l")) + .SetPublicId(SqidEncoder.Encode(12))) // Filter Option added + .Generate(4))) + .GenerateList(2); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + filterMetas: newFilterMeta); + + await CreateChanges(instanceId); + + var filterOptionMetaChanges = await GetFilterOptionMetaChanges(newVersion); + + // 12 Filter Option changes + Assert.Equal(12, filterOptionMetaChanges.Count); + Assert.All(filterOptionMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + var oldFilterMetas = originalVersion.FilterMetas + .ToDictionary( + m => m.PublicId, + m => m.OptionLinks.ToDictionary(l => l.PublicId)); + + var newFilterMetas = newVersion.FilterMetas + .ToDictionary( + m => m.PublicId, + m => m.OptionLinks.ToDictionary(l => l.PublicId)); + + // The changes should be inserted into each database table ordered alphabetically by 'Label'. + // They should also be ordered such that all deletions come first, updates next, and additions last. + + // Therefore, the expected order of Filter Option changes are (as per their Public IDs): + // Sqid 2 in filter with Sqid 1 deleted + // Sqid 5 in filter with Sqid 2 deleted + // Sqid 1 in filter with Sqid 1 deleted + // Sqid 6 in filter with Sqid 2 deleted + // Sqid 3 in filter with Sqid 1 updated + // Sqid 7 in filter with Sqid 2 updated + // Sqid 8 in filter with Sqid 2 updated + // Sqid 4 in filter with Sqid 1 updated + // Sqid 10 in filter with Sqid 1 added + // Sqid 11 in filter with Sqid 2 added + // Sqid 9 in filter with Sqid 1 added + // Sqid 12 in filter with Sqid 2 added + + AssertFilterOptionDeleted( + expectedOptionLink: oldFilterMetas[SqidEncoder.Encode(1)][SqidEncoder.Encode(2)], + change: filterOptionMetaChanges[0]); + AssertFilterOptionDeleted( + expectedOptionLink: oldFilterMetas[SqidEncoder.Encode(2)][SqidEncoder.Encode(5)], + change: filterOptionMetaChanges[1]); + AssertFilterOptionDeleted( + expectedOptionLink: oldFilterMetas[SqidEncoder.Encode(1)][SqidEncoder.Encode(1)], + change: filterOptionMetaChanges[2]); + AssertFilterOptionDeleted( + expectedOptionLink: oldFilterMetas[SqidEncoder.Encode(2)][SqidEncoder.Encode(6)], + change: filterOptionMetaChanges[3]); + AssertFilterOptionUpdated( + expectedOldOptionLink: oldFilterMetas[SqidEncoder.Encode(1)][SqidEncoder.Encode(3)], + expectedNewOptionLink: newFilterMetas[SqidEncoder.Encode(1)][SqidEncoder.Encode(3)], + change: filterOptionMetaChanges[4]); + AssertFilterOptionUpdated( + expectedOldOptionLink: oldFilterMetas[SqidEncoder.Encode(2)][SqidEncoder.Encode(7)], + expectedNewOptionLink: newFilterMetas[SqidEncoder.Encode(2)][SqidEncoder.Encode(7)], + change: filterOptionMetaChanges[5]); + AssertFilterOptionUpdated( + expectedOldOptionLink: oldFilterMetas[SqidEncoder.Encode(2)][SqidEncoder.Encode(8)], + expectedNewOptionLink: newFilterMetas[SqidEncoder.Encode(2)][SqidEncoder.Encode(8)], + change: filterOptionMetaChanges[6]); + AssertFilterOptionUpdated( + expectedOldOptionLink: oldFilterMetas[SqidEncoder.Encode(1)][SqidEncoder.Encode(4)], + expectedNewOptionLink: newFilterMetas[SqidEncoder.Encode(1)][SqidEncoder.Encode(4)], + change: filterOptionMetaChanges[7]); + AssertFilterOptionAdded( + expectedOptionLink: newFilterMetas[SqidEncoder.Encode(1)][SqidEncoder.Encode(10)], + change: filterOptionMetaChanges[8]); + AssertFilterOptionAdded( + expectedOptionLink: newFilterMetas[SqidEncoder.Encode(2)][SqidEncoder.Encode(11)], + change: filterOptionMetaChanges[9]); + AssertFilterOptionAdded( + expectedOptionLink: newFilterMetas[SqidEncoder.Encode(1)][SqidEncoder.Encode(9)], + change: filterOptionMetaChanges[10]); + AssertFilterOptionAdded( + expectedOptionLink: newFilterMetas[SqidEncoder.Encode(2)][SqidEncoder.Encode(12)], + change: filterOptionMetaChanges[11]); + } + + private static void AssertSingleFilterDeleted( + IReadOnlyList changes, + FilterMeta expectedFilter) + { + Assert.Single(changes, + c => c.PreviousStateId == expectedFilter.Id + && c.CurrentStateId is null); + } + + private static void AssertSingleFilterAdded( + IReadOnlyList changes, + FilterMeta expectedFilter) + { + Assert.Single(changes, + c => c.PreviousStateId is null + && c.CurrentStateId == expectedFilter.Id); + } + + private static void AssertSingleFilterUpdated( + IReadOnlyList changes, + FilterMeta expectedOldFilter, + FilterMeta expectedNewFilter) + { + Assert.Single(changes, + c => c.PreviousStateId == expectedOldFilter.Id + && c.CurrentStateId == expectedNewFilter.Id); + } + + private static void AssertFilterDeleted(FilterMeta expectedFilter, FilterMetaChange change) + { + Assert.Equal(expectedFilter.Id, change.PreviousStateId); + Assert.Null(change.CurrentStateId); + } + + private static void AssertFilterAdded(FilterMeta expectedFilter, FilterMetaChange change) + { + Assert.Null(change.PreviousStateId); + Assert.Equal(expectedFilter.Id, change.CurrentStateId); + } + + private static void AssertFilterUpdated( + FilterMeta expectedOldFilter, + FilterMeta expectedNewFilter, + FilterMetaChange change) + { + Assert.Equal(expectedOldFilter.Id, change.PreviousStateId); + Assert.Equal(expectedNewFilter.Id, change.CurrentStateId); + } + + private static void AssertSingleFilterOptionDeleted( + IReadOnlyList changes, + FilterOptionMetaLink expectedOptionLink) + { + Assert.Single(changes, + c => c.PreviousState!.PublicId == expectedOptionLink.PublicId + && c.PreviousState.MetaId == expectedOptionLink.MetaId + && c.PreviousState.OptionId == expectedOptionLink.OptionId + && c.CurrentState is null); + } + + private static void AssertSingleFilterOptionAdded( + IReadOnlyList changes, + FilterOptionMetaLink expectedOptionLink) + { + Assert.Single(changes, + c => c.PreviousState is null + && c.CurrentState!.PublicId == expectedOptionLink.PublicId + && c.CurrentState.MetaId == expectedOptionLink.MetaId + && c.CurrentState.OptionId == expectedOptionLink.OptionId); + } + + private static void AssertSingleFilterOptionUpdated( + IReadOnlyList changes, + FilterOptionMetaLink expectedOldOptionLink, + FilterOptionMetaLink expectedNewOptionLink) + { + Assert.Single(changes, + c => c.PreviousState!.PublicId == expectedOldOptionLink.PublicId + && c.PreviousState.MetaId == expectedOldOptionLink.MetaId + && c.PreviousState.OptionId == expectedOldOptionLink.OptionId + && c.CurrentState!.PublicId == expectedNewOptionLink.PublicId + && c.CurrentState.MetaId == expectedNewOptionLink.MetaId + && c.CurrentState.OptionId == expectedNewOptionLink.OptionId); + } + + private static void AssertFilterOptionDeleted( + FilterOptionMetaLink expectedOptionLink, + FilterOptionMetaChange change) + { + Assert.Equal(expectedOptionLink.PublicId, change.PreviousState!.PublicId); + Assert.Equal(expectedOptionLink.MetaId, change.PreviousState.MetaId); + Assert.Equal(expectedOptionLink.OptionId, change.PreviousState.OptionId); + Assert.Null(change.CurrentState); + } + + private static void AssertFilterOptionAdded( + FilterOptionMetaLink expectedOptionLink, + FilterOptionMetaChange change) + { + Assert.Null(change.PreviousState); + Assert.Equal(expectedOptionLink.PublicId, change.CurrentState!.PublicId); + Assert.Equal(expectedOptionLink.MetaId, change.CurrentState.MetaId); + Assert.Equal(expectedOptionLink.OptionId, change.CurrentState.OptionId); + } + + private static void AssertFilterOptionUpdated( + FilterOptionMetaLink expectedOldOptionLink, + FilterOptionMetaLink expectedNewOptionLink, + FilterOptionMetaChange change) + { + Assert.Equal(expectedOldOptionLink.PublicId, change.PreviousState!.PublicId); + Assert.Equal(expectedOldOptionLink.MetaId, change.PreviousState.MetaId); + Assert.Equal(expectedOldOptionLink.OptionId, change.PreviousState.OptionId); + Assert.Equal(expectedNewOptionLink.PublicId, change.CurrentState!.PublicId); + Assert.Equal(expectedNewOptionLink.MetaId, change.CurrentState.MetaId); + Assert.Equal(expectedNewOptionLink.OptionId, change.CurrentState.OptionId); + } + + private Action> UnchangedFilterMetaSetter( + FilterMeta filterMeta, + Func>? newOptionLinks = null) + { + return s => s + .SetPublicId(filterMeta.PublicId) + .SetColumn(filterMeta.Column) + .SetLabel(filterMeta.Label) + .SetHint(filterMeta.Hint) + .SetOptionLinks(newOptionLinks ??= () => + { + return filterMeta.OptionLinks + .Select(l => DataFixture.DefaultFilterOptionMetaLink() + .ForInstance(UnchangedFilterOptionMetaLinkSetter(l)) + .Generate()); + }); + } + + private static Action> UnchangedFilterOptionMetaLinkSetter( + FilterOptionMetaLink filterOptionMetaLink) + { + return s => s + .SetPublicId(filterOptionMetaLink.PublicId) + .SetOptionId(filterOptionMetaLink.OptionId); + } + + private async Task> GetFilterMetaChanges(DataSetVersion version) + { + return await GetDbContext() + .FilterMetaChanges + .AsNoTracking() + .Where(c => c.DataSetVersionId == version.Id) + .OrderBy(c => c.Id) + .ToListAsync(); + } + + private async Task> GetFilterOptionMetaChanges(DataSetVersion version) + { + return await GetDbContext() + .FilterOptionMetaChanges + .AsNoTracking() + .Where(c => c.DataSetVersionId == version.Id) + .OrderBy(c => c.Id) + .ToListAsync(); + } + + private async Task<(DataSetVersion originalVersion, Guid instanceId)> CreateDataSetInitialVersion( + List filterMetas) + { + return await CreateDataSetInitialVersion( + dataSetStatus: DataSetStatus.Published, + dataSetVersionStatus: DataSetVersionStatus.Published, + importStage: DataSetVersionImportStage.Completing, + meta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta(), + FilterMetas = filterMetas + }); + } + + private async Task<(DataSetVersion nextVersion, Guid instanceId)> CreateDataSetNextVersion( + DataSetVersion originalVersion, + List filterMetas) + { + return await CreateDataSetNextVersion( + initialVersion: originalVersion, + status: DataSetVersionStatus.Mapping, + importStage: Stage.PreviousStage(), + meta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta(), + FilterMetas = filterMetas + }); + } + } + + public class CreateChangesLocationTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : CreateChangesTests(fixture) + { + [Fact] + public async Task LocationsAdded_ChangesContainOnlyAddedLocations() + { + var oldLocationMeta = DataFixture.DefaultLocationMeta() + .ForIndex(0, s => + s.SetLevel(GeographicLevel.LocalAuthority) + .SetOptionLinks(() => + DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultLocationLocalAuthorityOptionMeta()) + .SetPublicId(SqidEncoder.Encode(1))) + .Generate(1))) + .GenerateList(1); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldLocationMeta); + + var newLocationMeta = DataFixture.DefaultLocationMeta() + .ForIndex(0, UnchangedLocationMetaSetter(oldLocationMeta[0])) // Location and ALL options unchanged + .ForIndex(1, s => + s.SetLevel(GeographicLevel.School) // Location added + .SetOptionLinks(() => + DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultLocationSchoolOptionMeta()) + .SetPublicId(SqidEncoder.Encode(2))) // Location Option added + .Generate(1))) + .ForIndex(2, s => + s.SetLevel(GeographicLevel.RscRegion) // Location added + .SetOptionLinks(() => + DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultLocationRscRegionOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) // Location Option added + .Generate(1))) + .GenerateList(3); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + locationMetas: newLocationMeta); + + await CreateChanges(instanceId); + + var locationMetaChanges = await GetLocationMetaChanges(newVersion); + var locationOptionMetaChanges = await GetLocationOptionMetaChanges(newVersion); + + // 2 Location additions + Assert.Equal(2, locationMetaChanges.Count); + Assert.All(locationMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + // No Location Option additions + Assert.Empty(locationOptionMetaChanges); + + var newLocationMetas = newVersion.LocationMetas + .ToDictionary(m => m.Level); + + // Location added + AssertSingleLocationAdded(locationMetaChanges, newLocationMetas[GeographicLevel.School]); + + // Location added + AssertSingleLocationAdded(locationMetaChanges, newLocationMetas[GeographicLevel.RscRegion]); + } + + [Fact] + public async Task LocationsDeleted_ChangesContainOnlyDeletedLocations() + { + var oldLocationMeta = DataFixture.DefaultLocationMeta() + .ForIndex(0, s => + s.SetLevel(GeographicLevel.LocalAuthority) + .SetOptionLinks(() => + DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultLocationLocalAuthorityOptionMeta()) + .SetPublicId(SqidEncoder.Encode(1))) + .Generate(1))) + .ForIndex(1, s => + s.SetLevel(GeographicLevel.School) // Location and ALL options deleted + .SetOptionLinks(() => + DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultLocationSchoolOptionMeta()) + .SetPublicId(SqidEncoder.Encode(2))) + .Generate(1))) + .ForIndex(2, s => + s.SetLevel(GeographicLevel.RscRegion) // Location and ALL options deleted + .SetOptionLinks(() => + DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultLocationRscRegionOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) + .Generate(1))) + .GenerateList(3); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldLocationMeta); + + var newLocationMeta = DataFixture.DefaultLocationMeta() + .ForIndex(0, UnchangedLocationMetaSetter(oldLocationMeta[0])) // Location and ALL options unchanged + .GenerateList(1); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + locationMetas: newLocationMeta); + + await CreateChanges(instanceId); + + var locationMetaChanges = await GetLocationMetaChanges(newVersion); + var locationOptionMetaChanges = await GetLocationOptionMetaChanges(newVersion); + + // 2 Location deletions + Assert.Equal(2, locationMetaChanges.Count); + Assert.All(locationMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + // No Location Option deletions + Assert.Empty(locationOptionMetaChanges); + + var oldLocationMetas = originalVersion.LocationMetas + .ToDictionary(m => m.Level); + + // Location deleted + AssertSingleLocationDeleted(locationMetaChanges, oldLocationMetas[GeographicLevel.School]); + + // Location deleted + AssertSingleLocationDeleted(locationMetaChanges, oldLocationMetas[GeographicLevel.RscRegion]); + } + + [Fact] + public async Task LocationsUnchangedOptionsAdded_ChangesContainOnlyAddedOptions() + { + var oldLocationMeta = DataFixture.DefaultLocationMeta() + .ForIndex(0, s => + s.SetLevel(GeographicLevel.LocalAuthority) + .SetOptionLinks(() => + DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultLocationLocalAuthorityOptionMeta()) + .SetPublicId(SqidEncoder.Encode(1))) + .Generate(1))) + .ForIndex(1, s => + s.SetLevel(GeographicLevel.School) + .SetOptionLinks(() => + DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultLocationSchoolOptionMeta()) + .SetPublicId(SqidEncoder.Encode(2))) + .Generate(1))) + .GenerateList(2); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldLocationMeta); + + var newLocationMeta = DataFixture.DefaultLocationMeta() + .ForIndex(0, UnchangedLocationMetaSetter( + locationMeta: oldLocationMeta[0], // Location unchanged + newOptionLinks: () => DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, + UnchangedLocationOptionMetaLinkSetter(oldLocationMeta[0] + .OptionLinks[0])) // Location Option unchanged + .ForIndex(1, s => + s.SetOption(DataFixture.DefaultLocationLocalAuthorityOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) // Location Option added + .ForIndex(2, s => + s.SetOption(DataFixture.DefaultLocationLocalAuthorityOptionMeta()) + .SetPublicId(SqidEncoder.Encode(4))) // Location Option added + .Generate(3))) + .ForIndex(1, UnchangedLocationMetaSetter( + locationMeta: oldLocationMeta[1], // Location unchanged + newOptionLinks: () => DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, + UnchangedLocationOptionMetaLinkSetter(oldLocationMeta[1] + .OptionLinks[0])) // Location Option unchanged + .ForIndex(1, s => + s.SetOption(DataFixture.DefaultLocationSchoolOptionMeta()) + .SetPublicId(SqidEncoder.Encode(5))) // Location Option added + .ForIndex(2, s => + s.SetOption(DataFixture.DefaultLocationSchoolOptionMeta()) + .SetPublicId(SqidEncoder.Encode(6))) // Location Option added + .Generate(3))) + .GenerateList(2); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + locationMetas: newLocationMeta); + + await CreateChanges(instanceId); + + var locationMetaChanges = await GetLocationMetaChanges(newVersion); + var locationOptionMetaChanges = await GetLocationOptionMetaChanges(newVersion); + + // No Location changes + Assert.Empty(locationMetaChanges); + + // 4 Location Option changes + Assert.Equal(4, locationOptionMetaChanges.Count); + Assert.All(locationOptionMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + var newLocationMetas = newVersion.LocationMetas + .ToDictionary( + m => m.Level, + m => m.OptionLinks.ToDictionary(l => l.PublicId)); + + // Location Option added + AssertSingleLocationOptionAdded(locationOptionMetaChanges, + newLocationMetas[GeographicLevel.LocalAuthority][SqidEncoder.Encode(3)]); + + // Location Option added + AssertSingleLocationOptionAdded(locationOptionMetaChanges, + newLocationMetas[GeographicLevel.LocalAuthority][SqidEncoder.Encode(4)]); + + // Location Option added + AssertSingleLocationOptionAdded(locationOptionMetaChanges, + newLocationMetas[GeographicLevel.School][SqidEncoder.Encode(5)]); + + // Location Option added + AssertSingleLocationOptionAdded(locationOptionMetaChanges, + newLocationMetas[GeographicLevel.School][SqidEncoder.Encode(6)]); + } + + [Fact] + public async Task LocationsUnchangedOptionsDeleted_ChangesContainOnlyDeletedOptions() + { + var oldLocationMeta = DataFixture.DefaultLocationMeta() + .ForIndex(0, s => + s.SetLevel(GeographicLevel.LocalAuthority) + .SetOptionLinks(() => + DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultLocationLocalAuthorityOptionMeta()) + .SetPublicId(SqidEncoder.Encode(1))) + .ForIndex(1, ls => + ls.SetOption(DataFixture.DefaultLocationLocalAuthorityOptionMeta()) + .SetPublicId(SqidEncoder.Encode(2))) // Location Option deleted + .ForIndex(2, ls => + ls.SetOption(DataFixture.DefaultLocationLocalAuthorityOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) // Location Option deleted + .Generate(3))) + .ForIndex(1, s => + s.SetLevel(GeographicLevel.School) + .SetOptionLinks(() => + DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultLocationSchoolOptionMeta()) + .SetPublicId(SqidEncoder.Encode(4))) + .ForIndex(1, ls => + ls.SetOption(DataFixture.DefaultLocationSchoolOptionMeta()) + .SetPublicId(SqidEncoder.Encode(5))) // Location Option deleted + .ForIndex(2, ls => + ls.SetOption(DataFixture.DefaultLocationSchoolOptionMeta()) + .SetPublicId(SqidEncoder.Encode(6))) // Location Option deleted + .Generate(3))) + .GenerateList(2); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldLocationMeta); + + var newLocationMeta = DataFixture.DefaultLocationMeta() + .ForIndex(0, UnchangedLocationMetaSetter( + locationMeta: oldLocationMeta[0], // Location unchanged + newOptionLinks: () => DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, + UnchangedLocationOptionMetaLinkSetter(oldLocationMeta[0] + .OptionLinks[0])) // Location Option unchanged + .Generate(1))) + .ForIndex(1, UnchangedLocationMetaSetter( + locationMeta: oldLocationMeta[1], // Location unchanged + newOptionLinks: () => DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, + UnchangedLocationOptionMetaLinkSetter(oldLocationMeta[1] + .OptionLinks[0])) // Location Option unchanged + .Generate(1))) + .GenerateList(2); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + locationMetas: newLocationMeta); + + await CreateChanges(instanceId); + + var locationMetaChanges = await GetLocationMetaChanges(newVersion); + var locationOptionMetaChanges = await GetLocationOptionMetaChanges(newVersion); + + // No Location changes + Assert.Empty(locationMetaChanges); + + // 4 Location Option changes + Assert.Equal(4, locationOptionMetaChanges.Count); + Assert.All(locationOptionMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + var oldLocationMetas = originalVersion.LocationMetas + .ToDictionary( + m => m.Level, + m => m.OptionLinks.ToDictionary(l => l.PublicId)); + + // Location Option deleted + AssertSingleLocationOptionDeleted(locationOptionMetaChanges, + oldLocationMetas[GeographicLevel.LocalAuthority][SqidEncoder.Encode(2)]); + + // Location Option deleted + AssertSingleLocationOptionDeleted(locationOptionMetaChanges, + oldLocationMetas[GeographicLevel.LocalAuthority][SqidEncoder.Encode(3)]); + + // Location Option deleted + AssertSingleLocationOptionDeleted(locationOptionMetaChanges, + oldLocationMetas[GeographicLevel.School][SqidEncoder.Encode(5)]); + + // Location Option deleted + AssertSingleLocationOptionDeleted(locationOptionMetaChanges, + oldLocationMetas[GeographicLevel.School][SqidEncoder.Encode(6)]); + } + + [Fact] + public async Task LocationsUnchangedOptionsUpdated_ChangesContainOnlyUpdatedOptions() + { + var oldLocationMeta = DataFixture.DefaultLocationMeta() + .ForIndex(0, s => + s.SetLevel(GeographicLevel.LocalAuthority) + .SetOptionLinks(() => + DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultLocationLocalAuthorityOptionMeta()) + .SetPublicId(SqidEncoder.Encode(1))) + .ForIndex(1, ls => + ls.SetOption(DataFixture.DefaultLocationLocalAuthorityOptionMeta()) + .SetPublicId(SqidEncoder.Encode(2))) + .ForIndex(2, ls => + ls.SetOption(DataFixture.DefaultLocationLocalAuthorityOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) + .Generate(3))) + .ForIndex(1, s => + s.SetLevel(GeographicLevel.School) + .SetOptionLinks(() => + DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultLocationSchoolOptionMeta()) + .SetPublicId(SqidEncoder.Encode(4))) + .ForIndex(1, ls => + ls.SetOption(DataFixture.DefaultLocationSchoolOptionMeta()) + .SetPublicId(SqidEncoder.Encode(5))) + .ForIndex(2, ls => + ls.SetOption(DataFixture.DefaultLocationSchoolOptionMeta()) + .SetPublicId(SqidEncoder.Encode(6))) + .Generate(3))) + .GenerateList(2); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldLocationMeta); + + var newLocationMeta = DataFixture.DefaultLocationMeta() + .ForIndex(0, UnchangedLocationMetaSetter( + locationMeta: oldLocationMeta[0], // Location unchanged + newOptionLinks: () => DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, + UnchangedLocationOptionMetaLinkSetter(oldLocationMeta[0] + .OptionLinks[0])) // Location Option unchanged + .ForIndex(1, s => + s.SetOption(DataFixture.DefaultLocationLocalAuthorityOptionMeta()) + .SetPublicId(SqidEncoder.Encode(2))) // Location Option updated + .ForIndex(2, s => + s.SetOption(DataFixture.DefaultLocationLocalAuthorityOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) // Location Option updated + .Generate(3))) + .ForIndex(1, UnchangedLocationMetaSetter( + locationMeta: oldLocationMeta[1], // Location unchanged + newOptionLinks: () => DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, + UnchangedLocationOptionMetaLinkSetter(oldLocationMeta[1] + .OptionLinks[0])) // Location Option unchanged + .ForIndex(1, s => + s.SetOption(DataFixture.DefaultLocationSchoolOptionMeta()) + .SetPublicId(SqidEncoder.Encode(5))) // Location Option updated + .ForIndex(2, s => + s.SetOption(DataFixture.DefaultLocationSchoolOptionMeta()) + .SetPublicId(SqidEncoder.Encode(6))) // Location Option updated + .Generate(3))) + .GenerateList(2); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + locationMetas: newLocationMeta); + + await CreateChanges(instanceId); + + var locationMetaChanges = await GetLocationMetaChanges(newVersion); + var locationOptionMetaChanges = await GetLocationOptionMetaChanges(newVersion); + + // No Location changes + Assert.Empty(locationMetaChanges); + + // 4 Location Option changes + Assert.Equal(4, locationOptionMetaChanges.Count); + Assert.All(locationOptionMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + var oldLocationMetas = originalVersion.LocationMetas + .ToDictionary( + m => m.Level, + m => m.OptionLinks.ToDictionary(l => l.PublicId)); + + var newLocationMetas = newVersion.LocationMetas + .ToDictionary( + m => m.Level, + m => m.OptionLinks.ToDictionary(l => l.PublicId)); + + // Location Option updated + AssertSingleLocationOptionUpdated( + changes: locationOptionMetaChanges, + expectedOldOptionLink: oldLocationMetas[GeographicLevel.LocalAuthority][SqidEncoder.Encode(2)], + expectedNewOptionLink: newLocationMetas[GeographicLevel.LocalAuthority][SqidEncoder.Encode(2)]); + + // Location Option updated + AssertSingleLocationOptionUpdated( + changes: locationOptionMetaChanges, + expectedOldOptionLink: oldLocationMetas[GeographicLevel.LocalAuthority][SqidEncoder.Encode(3)], + expectedNewOptionLink: newLocationMetas[GeographicLevel.LocalAuthority][SqidEncoder.Encode(3)]); + + // Location Option updated + AssertSingleLocationOptionUpdated( + changes: locationOptionMetaChanges, + expectedOldOptionLink: oldLocationMetas[GeographicLevel.School][SqidEncoder.Encode(5)], + expectedNewOptionLink: newLocationMetas[GeographicLevel.School][SqidEncoder.Encode(5)]); + + // Location Option updated + AssertSingleLocationOptionUpdated( + changes: locationOptionMetaChanges, + expectedOldOptionLink: oldLocationMetas[GeographicLevel.School][SqidEncoder.Encode(6)], + expectedNewOptionLink: newLocationMetas[GeographicLevel.School][SqidEncoder.Encode(6)]); + } + + [Fact] + public async Task LocationsUnchangedOptionsUnchanged_ChangesAreEmpty() + { + var oldLocationMeta = DataFixture.DefaultLocationMeta() + .ForIndex(0, s => + s.SetLevel(GeographicLevel.LocalAuthority) + .SetOptionLinks(() => + DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultLocationLocalAuthorityOptionMeta()) + .SetPublicId(SqidEncoder.Encode(1))) + .Generate(1))) + .ForIndex(1, s => + s.SetLevel(GeographicLevel.School) + .SetOptionLinks(() => + DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption(DataFixture.DefaultLocationSchoolOptionMeta()) + .SetPublicId(SqidEncoder.Encode(2))) + .Generate(1))) + .GenerateList(2); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldLocationMeta); + + var newLocationMeta = DataFixture.DefaultLocationMeta() + .ForIndex(0, UnchangedLocationMetaSetter(oldLocationMeta[0])) // Location and ALL options unchanged + .ForIndex(1, UnchangedLocationMetaSetter(oldLocationMeta[1])) // Location and ALL options unchanged + .GenerateList(2); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + locationMetas: newLocationMeta); + + await CreateChanges(instanceId); + + var locationMetaChanges = await GetLocationMetaChanges(newVersion); + var locationOptionMetaChanges = await GetLocationOptionMetaChanges(newVersion); + + // No Location changes + Assert.Empty(locationMetaChanges); + + // No Location Option changes + Assert.Empty(locationOptionMetaChanges); + } + + [Fact] + public async Task LocationsAddedAndDeleted_ChangesInsertedIntoDatabaseInCorrectOrder() + { + var oldLocationMeta = DataFixture.DefaultLocationMeta() + .ForIndex(0, s => s.SetLevel(GeographicLevel.School)) // Location deleted + .ForIndex(1, s => s.SetLevel(GeographicLevel.LocalAuthority)) // Location deleted + .GenerateList(2); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldLocationMeta); + + var newLocationMeta = DataFixture.DefaultLocationMeta() + .ForIndex(0, s => s.SetLevel(GeographicLevel.RscRegion)) // Location added + .ForIndex(1, s => s.SetLevel(GeographicLevel.Provider)) // Location added + .GenerateList(2); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + locationMetas: newLocationMeta); + + await CreateChanges(instanceId); + + var locationMetaChanges = await GetLocationMetaChanges(newVersion); + + // 4 Location changes + Assert.Equal(4, locationMetaChanges.Count); + Assert.All(locationMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + var oldLocationMetas = originalVersion.LocationMetas + .ToDictionary(m => m.Level); + + var newLocationMetas = newVersion.LocationMetas + .ToDictionary(m => m.Level); + + // The changes should be inserted into each database table ordered alphabetically by 'Level'. + // They should also be ordered such that all additions come last. + + // Therefore, the expected order of Location changes are (as per their Geographic Level): + // LocalAuthority deleted + // School deleted + // Provider added + // RscRegion added + + AssertLocationDeleted( + expectedLocation: oldLocationMetas[GeographicLevel.LocalAuthority], + change: locationMetaChanges[0]); + AssertLocationDeleted( + expectedLocation: oldLocationMetas[GeographicLevel.School], + change: locationMetaChanges[1]); + AssertLocationAdded( + expectedLocation: newLocationMetas[GeographicLevel.Provider], + change: locationMetaChanges[2]); + AssertLocationAdded( + expectedLocation: newLocationMetas[GeographicLevel.RscRegion], + change: locationMetaChanges[3]); + } + + [Fact] + public async Task LocationsOptionsAddedAndDeletedAndUpdated_ChangesInsertedIntoDatabaseInCorrectOrder() + { + var oldLocationMeta = DataFixture.DefaultLocationMeta() + .ForIndex(0, s => + s.SetLevel(GeographicLevel.EnglishDevolvedArea) + .SetOptionLinks(() => + DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption( + DataFixture.DefaultLocationCodedOptionMeta() + .WithLabel("d")) + .SetPublicId(SqidEncoder.Encode(1))) // Location Option deleted + .ForIndex(1, ls => + ls.SetOption( + DataFixture.DefaultLocationCodedOptionMeta() + .WithLabel("a")) + .SetPublicId(SqidEncoder.Encode(2))) // Location Option deleted + .ForIndex(2, ls => + ls.SetOption( + DataFixture.DefaultLocationCodedOptionMeta() + .WithLabel("b")) + .SetPublicId(SqidEncoder.Encode(3))) + .ForIndex(3, ls => + ls.SetOption( + DataFixture.DefaultLocationCodedOptionMeta() + .WithLabel("k")) + .SetPublicId(SqidEncoder.Encode(4))) + .Generate(4))) + .ForIndex(1, s => + s.SetLevel(GeographicLevel.Country) + .SetOptionLinks(() => + DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, ls => + ls.SetOption( + DataFixture.DefaultLocationCodedOptionMeta() + .WithLabel("c")) + .SetPublicId(SqidEncoder.Encode(5))) // Location Option deleted + .ForIndex(1, ls => + ls.SetOption( + DataFixture.DefaultLocationCodedOptionMeta() + .WithLabel("h")) + .SetPublicId(SqidEncoder.Encode(6))) // Location Option deleted + .ForIndex(2, ls => + ls.SetOption( + DataFixture.DefaultLocationCodedOptionMeta() + .WithLabel("f")) + .SetPublicId(SqidEncoder.Encode(7))) + .ForIndex(3, ls => + ls.SetOption( + DataFixture.DefaultLocationCodedOptionMeta() + .WithLabel("i")) + .SetPublicId(SqidEncoder.Encode(8))) + .Generate(4))) + .GenerateList(2); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldLocationMeta); + + var newLocationMeta = DataFixture.DefaultLocationMeta() + .ForIndex(0, UnchangedLocationMetaSetter( + locationMeta: oldLocationMeta[0], + newOptionLinks: () => DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, s => + s.SetOption(DataFixture.DefaultLocationCodedOptionMeta()) + .SetPublicId(SqidEncoder.Encode(3))) // Location Option updated + .ForIndex(1, s => + s.SetOption(DataFixture.DefaultLocationCodedOptionMeta()) + .SetPublicId(SqidEncoder.Encode(4))) // Location Option updated + .ForIndex(2, s => + s.SetOption( + DataFixture.DefaultLocationCodedOptionMeta() + .WithLabel("j")) + .SetPublicId(SqidEncoder.Encode(9))) // Location Option added + .ForIndex(3, s => + s.SetOption( + DataFixture.DefaultLocationCodedOptionMeta() + .WithLabel("e")) + .SetPublicId(SqidEncoder.Encode(10))) // Location Option added + .Generate(4))) + .ForIndex(1, UnchangedLocationMetaSetter( + locationMeta: oldLocationMeta[1], + newOptionLinks: () => DataFixture.DefaultLocationOptionMetaLink() + .ForIndex(0, s => + s.SetOption(DataFixture.DefaultLocationCodedOptionMeta()) + .SetPublicId(SqidEncoder.Encode(7))) // Location Option updated + .ForIndex(1, s => + s.SetOption(DataFixture.DefaultLocationCodedOptionMeta()) + .SetPublicId(SqidEncoder.Encode(8))) // Location Option updated + .ForIndex(2, s => + s.SetOption( + DataFixture.DefaultLocationCodedOptionMeta() + .WithLabel("g")) + .SetPublicId(SqidEncoder.Encode(11))) // Location Option added + .ForIndex(3, s => + s.SetOption( + DataFixture.DefaultLocationCodedOptionMeta() + .WithLabel("l")) + .SetPublicId(SqidEncoder.Encode(12))) // Location Option added + .Generate(4))) + .GenerateList(2); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + locationMetas: newLocationMeta); + + await CreateChanges(instanceId); + + var locationOptionMetaChanges = await GetLocationOptionMetaChanges(newVersion); + + // 12 Location Option changes + Assert.Equal(12, locationOptionMetaChanges.Count); + Assert.All(locationOptionMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + var oldLocationMetas = originalVersion.LocationMetas + .ToDictionary( + m => m.Level, + m => m.OptionLinks.ToDictionary(l => l.PublicId)); + + var newLocationMetas = newVersion.LocationMetas + .ToDictionary( + m => m.Level, + m => m.OptionLinks.ToDictionary(l => l.PublicId)); + + // The changes should be inserted into each database table ordered alphabetically by 'Label'. + // They should also be ordered such that all deletions come first, updates next, and additions last. + + // Therefore, the expected order of Location Option changes are (as per their Public IDs): + // Sqid 2 in Location with Level EnglishDevolvedArea deleted + // Sqid 5 in Location with Level Country deleted + // Sqid 1 in Location with Level EnglishDevolvedArea deleted + // Sqid 6 in Location with Level Country deleted + // Sqid 3 in Location with Level EnglishDevolvedArea updated + // Sqid 7 in Location with Level Country updated + // Sqid 8 in Location with Level Country updated + // Sqid 4 in Location with Level EnglishDevolvedArea updated + // Sqid 10 in Location with Level EnglishDevolvedArea added + // Sqid 11 in Location with Level Country added + // Sqid 9 in Location with Level EnglishDevolvedArea added + // Sqid 12 in Location with Level Country added + + AssertLocationOptionDeleted( + expectedOptionLink: oldLocationMetas[GeographicLevel.EnglishDevolvedArea][SqidEncoder.Encode(2)], + change: locationOptionMetaChanges[0]); + AssertLocationOptionDeleted( + expectedOptionLink: oldLocationMetas[GeographicLevel.Country][SqidEncoder.Encode(5)], + change: locationOptionMetaChanges[1]); + AssertLocationOptionDeleted( + expectedOptionLink: oldLocationMetas[GeographicLevel.EnglishDevolvedArea][SqidEncoder.Encode(1)], + change: locationOptionMetaChanges[2]); + AssertLocationOptionDeleted( + expectedOptionLink: oldLocationMetas[GeographicLevel.Country][SqidEncoder.Encode(6)], + change: locationOptionMetaChanges[3]); + AssertLocationOptionUpdated( + expectedOldOptionLink: oldLocationMetas[GeographicLevel.EnglishDevolvedArea][SqidEncoder.Encode(3)], + expectedNewOptionLink: newLocationMetas[GeographicLevel.EnglishDevolvedArea][SqidEncoder.Encode(3)], + change: locationOptionMetaChanges[4]); + AssertLocationOptionUpdated( + expectedOldOptionLink: oldLocationMetas[GeographicLevel.Country][SqidEncoder.Encode(7)], + expectedNewOptionLink: newLocationMetas[GeographicLevel.Country][SqidEncoder.Encode(7)], + change: locationOptionMetaChanges[5]); + AssertLocationOptionUpdated( + expectedOldOptionLink: oldLocationMetas[GeographicLevel.Country][SqidEncoder.Encode(8)], + expectedNewOptionLink: newLocationMetas[GeographicLevel.Country][SqidEncoder.Encode(8)], + change: locationOptionMetaChanges[6]); + AssertLocationOptionUpdated( + expectedOldOptionLink: oldLocationMetas[GeographicLevel.EnglishDevolvedArea][SqidEncoder.Encode(4)], + expectedNewOptionLink: newLocationMetas[GeographicLevel.EnglishDevolvedArea][SqidEncoder.Encode(4)], + change: locationOptionMetaChanges[7]); + AssertLocationOptionAdded( + expectedOptionLink: newLocationMetas[GeographicLevel.EnglishDevolvedArea][SqidEncoder.Encode(10)], + change: locationOptionMetaChanges[8]); + AssertLocationOptionAdded( + expectedOptionLink: newLocationMetas[GeographicLevel.Country][SqidEncoder.Encode(11)], + change: locationOptionMetaChanges[9]); + AssertLocationOptionAdded( + expectedOptionLink: newLocationMetas[GeographicLevel.EnglishDevolvedArea][SqidEncoder.Encode(9)], + change: locationOptionMetaChanges[10]); + AssertLocationOptionAdded( + expectedOptionLink: newLocationMetas[GeographicLevel.Country][SqidEncoder.Encode(12)], + change: locationOptionMetaChanges[11]); + } + + private static void AssertSingleLocationDeleted( + IReadOnlyList changes, + LocationMeta expectedLocation) + { + Assert.Single(changes, + c => c.PreviousStateId == expectedLocation.Id + && c.CurrentStateId is null); + } + + private static void AssertSingleLocationAdded( + IReadOnlyList changes, + LocationMeta expectedLocation) + { + Assert.Single(changes, + c => c.PreviousStateId is null + && c.CurrentStateId == expectedLocation.Id); + } + + private static void AssertSingleLocationUpdated( + IReadOnlyList changes, + LocationMeta expectedOldLocation, + LocationMeta expectedNewLocation) + { + Assert.Single(changes, + c => c.PreviousStateId == expectedOldLocation.Id + && c.CurrentStateId == expectedNewLocation.Id); + } + + private static void AssertLocationDeleted(LocationMeta expectedLocation, LocationMetaChange change) + { + Assert.Equal(expectedLocation.Id, change.PreviousStateId); + Assert.Null(change.CurrentStateId); + } + + private static void AssertLocationAdded(LocationMeta expectedLocation, LocationMetaChange change) + { + Assert.Null(change.PreviousStateId); + Assert.Equal(expectedLocation.Id, change.CurrentStateId); + } + + private static void AssertSingleLocationOptionDeleted( + IReadOnlyList changes, + LocationOptionMetaLink expectedOptionLink) + { + Assert.Single(changes, + c => c.PreviousState!.PublicId == expectedOptionLink.PublicId + && c.PreviousState.MetaId == expectedOptionLink.MetaId + && c.PreviousState.OptionId == expectedOptionLink.OptionId + && c.CurrentState is null); + } + + private static void AssertSingleLocationOptionAdded( + IReadOnlyList changes, + LocationOptionMetaLink expectedOptionLink) + { + Assert.Single(changes, + c => c.PreviousState is null + && c.CurrentState!.PublicId == expectedOptionLink.PublicId + && c.CurrentState.MetaId == expectedOptionLink.MetaId + && c.CurrentState.OptionId == expectedOptionLink.OptionId); + } + + private static void AssertSingleLocationOptionUpdated( + IReadOnlyList changes, + LocationOptionMetaLink expectedOldOptionLink, + LocationOptionMetaLink expectedNewOptionLink) + { + Assert.Single(changes, + c => c.PreviousState!.PublicId == expectedOldOptionLink.PublicId + && c.PreviousState.MetaId == expectedOldOptionLink.MetaId + && c.PreviousState.OptionId == expectedOldOptionLink.OptionId + && c.CurrentState!.PublicId == expectedNewOptionLink.PublicId + && c.CurrentState.MetaId == expectedNewOptionLink.MetaId + && c.CurrentState.OptionId == expectedNewOptionLink.OptionId); + } + + private static void AssertLocationOptionDeleted( + LocationOptionMetaLink expectedOptionLink, + LocationOptionMetaChange change) + { + Assert.Equal(expectedOptionLink.PublicId, change.PreviousState!.PublicId); + Assert.Equal(expectedOptionLink.MetaId, change.PreviousState.MetaId); + Assert.Equal(expectedOptionLink.OptionId, change.PreviousState.OptionId); + Assert.Null(change.CurrentState); + } + + private static void AssertLocationOptionAdded( + LocationOptionMetaLink expectedOptionLink, + LocationOptionMetaChange change) + { + Assert.Null(change.PreviousState); + Assert.Equal(expectedOptionLink.PublicId, change.CurrentState!.PublicId); + Assert.Equal(expectedOptionLink.MetaId, change.CurrentState.MetaId); + Assert.Equal(expectedOptionLink.OptionId, change.CurrentState.OptionId); + } + + private static void AssertLocationOptionUpdated( + LocationOptionMetaLink expectedOldOptionLink, + LocationOptionMetaLink expectedNewOptionLink, + LocationOptionMetaChange change) + { + Assert.Equal(expectedOldOptionLink.PublicId, change.PreviousState!.PublicId); + Assert.Equal(expectedOldOptionLink.MetaId, change.PreviousState.MetaId); + Assert.Equal(expectedOldOptionLink.OptionId, change.PreviousState.OptionId); + Assert.Equal(expectedNewOptionLink.PublicId, change.CurrentState!.PublicId); + Assert.Equal(expectedNewOptionLink.MetaId, change.CurrentState.MetaId); + Assert.Equal(expectedNewOptionLink.OptionId, change.CurrentState.OptionId); + } + + private Action> UnchangedLocationMetaSetter( + LocationMeta locationMeta, + Func>? newOptionLinks = null) + { + return s => s + .SetLevel(locationMeta.Level) + .SetOptionLinks(newOptionLinks ??= () => locationMeta + .OptionLinks + .Select(l => DataFixture.DefaultLocationOptionMetaLink() + .ForInstance(UnchangedLocationOptionMetaLinkSetter(l)) + .Generate() + )); + } + + private static Action> UnchangedLocationOptionMetaLinkSetter( + LocationOptionMetaLink locationOptionMetaLink) + { + return s => s + .SetPublicId(locationOptionMetaLink.PublicId) + .SetOptionId(locationOptionMetaLink.OptionId); + } + + private async Task> GetLocationMetaChanges(DataSetVersion version) + { + return await GetDbContext() + .LocationMetaChanges + .AsNoTracking() + .Where(c => c.DataSetVersionId == version.Id) + .OrderBy(c => c.Id) + .ToListAsync(); + } + + private async Task> GetLocationOptionMetaChanges(DataSetVersion version) + { + return await GetDbContext() + .LocationOptionMetaChanges + .AsNoTracking() + .Where(c => c.DataSetVersionId == version.Id) + .OrderBy(c => c.Id) + .ToListAsync(); + } + + private async Task<(DataSetVersion originalVersion, Guid instanceId)> CreateDataSetInitialVersion( + List locationMetas) + { + return await CreateDataSetInitialVersion( + dataSetStatus: DataSetStatus.Published, + dataSetVersionStatus: DataSetVersionStatus.Published, + importStage: DataSetVersionImportStage.Completing, + meta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta(), + LocationMetas = locationMetas + }); + } + + private async Task<(DataSetVersion nextVersion, Guid instanceId)> CreateDataSetNextVersion( + DataSetVersion originalVersion, + List locationMetas) + { + return await CreateDataSetNextVersion( + initialVersion: originalVersion, + status: DataSetVersionStatus.Mapping, + importStage: Stage.PreviousStage(), + meta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta(), + LocationMetas = locationMetas + }); + } + } + + public class CreateChangesGeographicLevelTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : CreateChangesTests(fixture) + { + [Fact] + public async Task GeographicLevelsAddedAndDeleted_ChangeExists() + { + var (originalVersion, newVersion, instanceId) = await CreateDataSetInitialAndNextVersion( + nextVersionStatus: DataSetVersionStatus.Mapping, + nextVersionImportStage: Stage.PreviousStage(), + initialVersionMeta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta() + .WithLevels([ + GeographicLevel.Country, + GeographicLevel.Region, + GeographicLevel.LocalAuthority + ]) + }, + nextVersionMeta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta() + .WithLevels([ + GeographicLevel.LocalAuthority, + GeographicLevel.LocalAuthorityDistrict, + GeographicLevel.School + ]) + }); + + await CreateChanges(instanceId); + + var actualChange = await GetDbContext() + .GeographicLevelMetaChanges + .AsNoTracking() + .SingleOrDefaultAsync(c => c.DataSetVersionId == newVersion.Id); + + Assert.NotNull(actualChange); + Assert.Equal(newVersion.Id, actualChange.DataSetVersionId); + Assert.Equal(originalVersion.GeographicLevelMeta!.Id, actualChange.PreviousStateId); + Assert.Equal(newVersion.GeographicLevelMeta!.Id, actualChange.CurrentStateId); + } + + [Fact] + public async Task GeographicLevelsUnchanged_ChangeIsNull() + { + var (originalVersion, newVersion, instanceId) = await CreateDataSetInitialAndNextVersion( + nextVersionStatus: DataSetVersionStatus.Mapping, + nextVersionImportStage: Stage.PreviousStage(), + initialVersionMeta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta() + .WithLevels([ + GeographicLevel.Country, + GeographicLevel.Region, + GeographicLevel.LocalAuthority + ]) + }, + nextVersionMeta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta() + .WithLevels([ + GeographicLevel.Country, + GeographicLevel.Region, + GeographicLevel.LocalAuthority + ]) + }); + + await CreateChanges(instanceId); + + var actualChange = await GetDbContext() + .GeographicLevelMetaChanges + .AsNoTracking() + .SingleOrDefaultAsync(c => c.DataSetVersionId == newVersion.Id); + + Assert.Null(actualChange); + } + + [Fact] + public async Task GeographicLevelsAdded_ChangeExists() + { + var (originalVersion, newVersion, instanceId) = await CreateDataSetInitialAndNextVersion( + nextVersionStatus: DataSetVersionStatus.Mapping, + nextVersionImportStage: Stage.PreviousStage(), + initialVersionMeta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta() + .WithLevels([ + GeographicLevel.Country + ]) + }, + nextVersionMeta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta() + .WithLevels([ + GeographicLevel.Country, + GeographicLevel.Region, + GeographicLevel.LocalAuthority + ]) + }); + + await CreateChanges(instanceId); + + var actualChange = await GetDbContext() + .GeographicLevelMetaChanges + .AsNoTracking() + .SingleOrDefaultAsync(c => c.DataSetVersionId == newVersion.Id); + + Assert.NotNull(actualChange); + Assert.Equal(newVersion.Id, actualChange.DataSetVersionId); + Assert.Equal(originalVersion.GeographicLevelMeta!.Id, actualChange.PreviousStateId); + Assert.Equal(newVersion.GeographicLevelMeta!.Id, actualChange.CurrentStateId); + } + + [Fact] + public async Task GeographicLevelsDeleted_ChangeExists() + { + var (originalVersion, newVersion, instanceId) = await CreateDataSetInitialAndNextVersion( + nextVersionStatus: DataSetVersionStatus.Mapping, + nextVersionImportStage: Stage.PreviousStage(), + initialVersionMeta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta() + .WithLevels([ + GeographicLevel.Country, + GeographicLevel.Region, + GeographicLevel.LocalAuthority + ]) + }, + nextVersionMeta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta() + .WithLevels([ + GeographicLevel.Country + ]) + }); + + await CreateChanges(instanceId); + + var actualChange = await GetDbContext() + .GeographicLevelMetaChanges + .AsNoTracking() + .SingleOrDefaultAsync(c => c.DataSetVersionId == newVersion.Id); + + Assert.NotNull(actualChange); + Assert.Equal(newVersion.Id, actualChange.DataSetVersionId); + Assert.Equal(originalVersion.GeographicLevelMeta!.Id, actualChange.PreviousStateId); + Assert.Equal(newVersion.GeographicLevelMeta!.Id, actualChange.CurrentStateId); + } + } + + public class CreateChangesIndicatorTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : CreateChangesTests(fixture) + { + [Fact] + public async Task IndicatorsAdded_ChangesContainOnlyAddedIndicators() + { + var oldIndicatorMeta = DataFixture.DefaultIndicatorMeta() + .ForIndex(0, s => s.SetPublicId(SqidEncoder.Encode(1))) + .GenerateList(1); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldIndicatorMeta); + + var newIndicatorMeta = DataFixture.DefaultIndicatorMeta() + .ForIndex(0, UnchangedIndicatorMetaSetter(oldIndicatorMeta[0])) // Indicator unchanged + .ForIndex(1, s => s.SetPublicId(SqidEncoder.Encode(2))) // Indicator added + .ForIndex(2, s => s.SetPublicId(SqidEncoder.Encode(3))) // Indicator added + .GenerateList(3); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + indicatorMetas: newIndicatorMeta); + + await CreateChanges(instanceId); + + var changes = await GetIndicatorMetaChanges(newVersion); + + // 2 Indicator changes + Assert.Equal(2, changes.Count); + Assert.All(changes, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + var newIndicatorMetas = newVersion.IndicatorMetas + .ToDictionary(m => m.PublicId); + + // Indicator added + AssertSingleIndicatorAdded(changes, newIndicatorMetas[SqidEncoder.Encode(2)]); + + // Indicator added + AssertSingleIndicatorAdded(changes, newIndicatorMetas[SqidEncoder.Encode(3)]); + } + + [Fact] + public async Task IndicatorsDeleted_ChangesContainOnlyDeletedIndicators() + { + var oldIndicatorMeta = DataFixture.DefaultIndicatorMeta() + .ForIndex(0, s => s.SetPublicId(SqidEncoder.Encode(1))) + .ForIndex(1, s => s.SetPublicId(SqidEncoder.Encode(2))) // Indicator deleted + .ForIndex(2, s => s.SetPublicId(SqidEncoder.Encode(3))) // Indicator deleted + .GenerateList(3); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldIndicatorMeta); + + var newIndicatorMeta = DataFixture.DefaultIndicatorMeta() + .ForIndex(0, UnchangedIndicatorMetaSetter(oldIndicatorMeta[0])) // Indicator unchanged + .GenerateList(1); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + indicatorMetas: newIndicatorMeta); + + await CreateChanges(instanceId); + + var changes = await GetIndicatorMetaChanges(newVersion); + + // 2 Indicator changes + Assert.Equal(2, changes.Count); + Assert.All(changes, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + var oldIndicatorMetas = originalVersion.IndicatorMetas + .ToDictionary(m => m.PublicId); + + // Indicator deleted + AssertSingleIndicatorDeleted(changes, oldIndicatorMetas[SqidEncoder.Encode(2)]); + + // Indicator deleted + AssertSingleIndicatorDeleted(changes, oldIndicatorMetas[SqidEncoder.Encode(3)]); + } + + [Fact] + public async Task IndicatorsUnchanged_ChangesAreEmpty() + { + var oldIndicatorMeta = DataFixture.DefaultIndicatorMeta() + .ForIndex(0, s => s.SetPublicId(SqidEncoder.Encode(1))) + .ForIndex(1, s => s.SetPublicId(SqidEncoder.Encode(2))) + .GenerateList(2); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldIndicatorMeta); + + var newIndicatorMeta = DataFixture.DefaultIndicatorMeta() + .ForIndex(0, UnchangedIndicatorMetaSetter(oldIndicatorMeta[0])) // Indicator unchanged + .ForIndex(1, UnchangedIndicatorMetaSetter(oldIndicatorMeta[1])) // Indicator unchanged + .GenerateList(2); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + indicatorMetas: newIndicatorMeta); + + await CreateChanges(instanceId); + + var changes = await GetIndicatorMetaChanges(newVersion); + + // No Indicator changes + Assert.Empty(changes); + } + + [Fact] + public async Task IndicatorsAddedAndDeletedAndUpdated_ChangesInsertedIntoDatabaseInCorrectOrder() + { + var oldIndicatorMeta = DataFixture.DefaultIndicatorMeta() + .ForIndex(0, s => + s.SetPublicId(SqidEncoder.Encode(1)) // Indicator deleted + .SetLabel("f")) + .ForIndex(1, s => + s.SetPublicId(SqidEncoder.Encode(2)) // Indicator deleted + .SetLabel("a")) + .ForIndex(2, s => + s.SetPublicId(SqidEncoder.Encode(3)) + .SetLabel("e")) + .ForIndex(3, s => + s.SetPublicId(SqidEncoder.Encode(4)) + .SetLabel("b")) + .GenerateList(4); + + var (originalVersion, _) = await CreateDataSetInitialVersion(oldIndicatorMeta); + + var newIndicatorMeta = DataFixture.DefaultIndicatorMeta() + .ForIndex(0, s => s.SetPublicId(SqidEncoder.Encode(3))) // Indicator updated + .ForIndex(1, s => s.SetPublicId(SqidEncoder.Encode(4))) // Indicator updated + .ForIndex(2, s => + s.SetPublicId(SqidEncoder.Encode(5)) // Indicator added + .SetLabel("d")) + .ForIndex(3, s => + s.SetPublicId(SqidEncoder.Encode(6)) // Indicator added + .SetLabel("c")) + .GenerateList(4); + + var (newVersion, instanceId) = await CreateDataSetNextVersion( + originalVersion: originalVersion, + indicatorMetas: newIndicatorMeta); + + await CreateChanges(instanceId); + + var indicatorMetaChanges = await GetIndicatorMetaChanges(newVersion); + + // 6 Indicator changes + Assert.Equal(6, indicatorMetaChanges.Count); + Assert.All(indicatorMetaChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + var oldIndicatorMetas = originalVersion.IndicatorMetas + .ToDictionary(m => m.PublicId); + + var newIndicatorMetas = newVersion.IndicatorMetas + .ToDictionary(m => m.PublicId); + + // The changes should be inserted into each database table ordered alphabetically by 'Label'. + // They should also be ordered such that all deletions come first, updates next, and additions last. + + // Therefore, the expected order of Indicator changes are (as per their Public IDs): + // Sqid 2 deleted + // Sqid 1 deleted + // Sqid 4 updated + // Sqid 3 updated + // Sqid 6 added + // Sqid 5 added + + AssertIndicatorDeleted( + expectedIndicator: oldIndicatorMetas[SqidEncoder.Encode(2)], + change: indicatorMetaChanges[0]); + AssertIndicatorDeleted( + expectedIndicator: oldIndicatorMetas[SqidEncoder.Encode(1)], + change: indicatorMetaChanges[1]); + AssertIndicatorUpdated( + expectedOldIndicator: oldIndicatorMetas[SqidEncoder.Encode(4)], + expectedNewIndicator: newIndicatorMetas[SqidEncoder.Encode(4)], + change: indicatorMetaChanges[2]); + AssertIndicatorUpdated( + expectedOldIndicator: oldIndicatorMetas[SqidEncoder.Encode(3)], + expectedNewIndicator: newIndicatorMetas[SqidEncoder.Encode(3)], + change: indicatorMetaChanges[3]); + AssertIndicatorAdded( + expectedIndicator: newIndicatorMetas[SqidEncoder.Encode(6)], + change: indicatorMetaChanges[4]); + AssertIndicatorAdded( + expectedIndicator: newIndicatorMetas[SqidEncoder.Encode(5)], + change: indicatorMetaChanges[5]); + } + + private static void AssertSingleIndicatorDeleted( + IReadOnlyList changes, + IndicatorMeta expectedIndicator) + { + Assert.Single(changes, + c => c.PreviousStateId == expectedIndicator.Id + && c.CurrentStateId is null); + } + + private static void AssertSingleIndicatorAdded( + IReadOnlyList changes, + IndicatorMeta expectedIndicator) + { + Assert.Single(changes, + c => c.PreviousStateId is null + && c.CurrentStateId == expectedIndicator.Id); + } + + private static void AssertSingleIndicatorUpdated( + IReadOnlyList changes, + IndicatorMeta expectedOldIndicator, + IndicatorMeta expectedNewIndicator) + { + Assert.Single(changes, + c => c.PreviousStateId == expectedOldIndicator.Id + && c.CurrentStateId == expectedNewIndicator.Id); + } + + private static void AssertIndicatorDeleted(IndicatorMeta expectedIndicator, IndicatorMetaChange change) + { + Assert.Equal(expectedIndicator.Id, change.PreviousStateId); + Assert.Null(change.CurrentStateId); + } + + private static void AssertIndicatorAdded(IndicatorMeta expectedIndicator, IndicatorMetaChange change) + { + Assert.Null(change.PreviousStateId); + Assert.Equal(expectedIndicator.Id, change.CurrentStateId); + } + + private static void AssertIndicatorUpdated( + IndicatorMeta expectedOldIndicator, + IndicatorMeta expectedNewIndicator, + IndicatorMetaChange change) + { + Assert.Equal(expectedOldIndicator.Id, change.PreviousStateId); + Assert.Equal(expectedNewIndicator.Id, change.CurrentStateId); + } + + private static Action> UnchangedIndicatorMetaSetter(IndicatorMeta indicatorMeta) + { + return s => s + .SetPublicId(indicatorMeta.PublicId) + .SetColumn(indicatorMeta.Column) + .SetLabel(indicatorMeta.Label) + .SetUnit(indicatorMeta.Unit) + .SetDecimalPlaces(indicatorMeta.DecimalPlaces); + } + + private async Task> GetIndicatorMetaChanges(DataSetVersion version) + { + return await GetDbContext() + .IndicatorMetaChanges + .AsNoTracking() + .Where(c => c.DataSetVersionId == version.Id) + .OrderBy(c => c.Id) + .ToListAsync(); + } + + private async Task<(DataSetVersion originalVersion, Guid instanceId)> CreateDataSetInitialVersion( + List indicatorMetas) + { + return await CreateDataSetInitialVersion( + dataSetStatus: DataSetStatus.Published, + dataSetVersionStatus: DataSetVersionStatus.Published, + importStage: DataSetVersionImportStage.Completing, + meta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta(), + IndicatorMetas = indicatorMetas + }); + } + + private async Task<(DataSetVersion nextVersion, Guid instanceId)> CreateDataSetNextVersion( + DataSetVersion originalVersion, + List indicatorMetas) + { + return await CreateDataSetNextVersion( + initialVersion: originalVersion, + status: DataSetVersionStatus.Mapping, + importStage: Stage.PreviousStage(), + meta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta(), + IndicatorMetas = indicatorMetas + }); + } + } + + public class CreateChangesTimePeriodTests( + ProcessorFunctionsIntegrationTestFixture fixture) + : CreateChangesTests(fixture) + { + [Fact] + public async Task TimePeriodsAddedAndDeleted_ChangesContainAdditionsAndDeletions() + { + var (originalVersion, newVersion, instanceId) = await CreateDataSetInitialAndNextVersion( + nextVersionStatus: DataSetVersionStatus.Mapping, + nextVersionImportStage: Stage.PreviousStage(), + initialVersionMeta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta(), + TimePeriodMetas = DataFixture.DefaultTimePeriodMeta() + .WithCode(TimeIdentifier.AcademicYear) + .ForIndex(0, s => s.SetPeriod("2020")) + .ForIndex(1, s => s.SetPeriod("2021")) + .ForIndex(2, s => s.SetPeriod("2022")) + .Generate(3) + }, + nextVersionMeta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta(), + TimePeriodMetas = DataFixture.DefaultTimePeriodMeta() + .WithCode(TimeIdentifier.AcademicYear) + .ForIndex(0, s => s.SetPeriod("2022")) + .ForIndex(1, s => s.SetPeriod("2023")) + .ForIndex(2, s => s.SetPeriod("2024")) + .Generate(3) + }); + + await CreateChanges(instanceId); + + var actualChanges = await GetDbContext() + .TimePeriodMetaChanges + .AsNoTracking() + .Where(c => c.DataSetVersionId == newVersion.Id) + .OrderBy(c => c.Id) + .ToListAsync(); + + Assert.Equal(4, actualChanges.Count); + Assert.All(actualChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + + var originalTimePeriodMetas = originalVersion.TimePeriodMetas + .ToDictionary(m => (m.Code, m.Period)); + + Assert.Equal(originalTimePeriodMetas[(TimeIdentifier.AcademicYear, "2020")].Id, + actualChanges[0].PreviousStateId); + Assert.Null(actualChanges[0].CurrentStateId); + + Assert.Equal(originalTimePeriodMetas[(TimeIdentifier.AcademicYear, "2021")].Id, + actualChanges[1].PreviousStateId); + Assert.Null(actualChanges[1].CurrentStateId); + + var newTimePeriodMetas = newVersion.TimePeriodMetas + .ToDictionary(m => (m.Code, m.Period)); + + Assert.Null(actualChanges[2].PreviousStateId); + Assert.Equal(newTimePeriodMetas[(TimeIdentifier.AcademicYear, "2023")].Id, actualChanges[2].CurrentStateId); + + Assert.Null(actualChanges[3].PreviousStateId); + Assert.Equal(newTimePeriodMetas[(TimeIdentifier.AcademicYear, "2024")].Id, actualChanges[3].CurrentStateId); + } + + [Fact] + public async Task TimePeriodsUnchanged_ChangesAreEmpty() + { + var (originalVersion, newVersion, instanceId) = await CreateDataSetInitialAndNextVersion( + nextVersionStatus: DataSetVersionStatus.Mapping, + nextVersionImportStage: Stage.PreviousStage(), + initialVersionMeta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta(), + TimePeriodMetas = DataFixture.DefaultTimePeriodMeta() + .WithCode(TimeIdentifier.AcademicYear) + .ForIndex(0, s => s.SetPeriod("2019").SetCode(TimeIdentifier.AcademicYear)) + .ForIndex(1, s => s.SetPeriod("2020").SetCode(TimeIdentifier.CalendarYear)) + .ForIndex(2, s => s.SetPeriod("2021").SetCode(TimeIdentifier.January)) + .Generate(3) + }, + nextVersionMeta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta(), + TimePeriodMetas = DataFixture.DefaultTimePeriodMeta() + .ForIndex(0, s => s.SetPeriod("2019").SetCode(TimeIdentifier.AcademicYear)) + .ForIndex(1, s => s.SetPeriod("2020").SetCode(TimeIdentifier.CalendarYear)) + .ForIndex(2, s => s.SetPeriod("2021").SetCode(TimeIdentifier.January)) + .Generate(3) + }); + + await CreateChanges(instanceId); + + Assert.False(await GetDbContext() + .TimePeriodMetaChanges + .AnyAsync(c => c.DataSetVersionId == newVersion.Id)); + } + + [Fact] + public async Task TimePeriodsAdded_ChangesContainOnlyAdditions() + { + var (originalVersion, newVersion, instanceId) = await CreateDataSetInitialAndNextVersion( + nextVersionStatus: DataSetVersionStatus.Mapping, + nextVersionImportStage: Stage.PreviousStage(), + initialVersionMeta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta(), + TimePeriodMetas = DataFixture.DefaultTimePeriodMeta() + .WithCode(TimeIdentifier.CalendarYear) + .WithPeriod("2020") + .Generate(1) + }, + nextVersionMeta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta(), + TimePeriodMetas = + [ + ..DataFixture.DefaultTimePeriodMeta() + .WithCode(TimeIdentifier.CalendarYear) + .WithPeriod("2020") + .Generate(1), + ..DataFixture.DefaultTimePeriodMeta() + .WithCode(TimeIdentifier.AcademicYear) + .ForIndex(0, s => s.SetPeriod("2019")) + .ForIndex(1, s => s.SetPeriod("2020")) + .ForIndex(2, s => s.SetPeriod("2021")) + .Generate(3) + ] + }); + + await CreateChanges(instanceId); + + var actualChanges = await GetDbContext() + .TimePeriodMetaChanges + .AsNoTracking() + .Where(c => c.DataSetVersionId == newVersion.Id) + .OrderBy(c => c.Id) + .ToListAsync(); + + Assert.Equal(3, actualChanges.Count); + Assert.All(actualChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + Assert.All(actualChanges, c => Assert.Null(c.PreviousStateId)); + + var newTimePeriodMetas = newVersion.TimePeriodMetas + .ToDictionary(m => (m.Code, m.Period)); + + Assert.Equal(newTimePeriodMetas[(TimeIdentifier.AcademicYear, "2019")].Id, actualChanges[0].CurrentStateId); + Assert.Equal(newTimePeriodMetas[(TimeIdentifier.AcademicYear, "2020")].Id, actualChanges[1].CurrentStateId); + Assert.Equal(newTimePeriodMetas[(TimeIdentifier.AcademicYear, "2021")].Id, actualChanges[2].CurrentStateId); + } + + [Fact] + public async Task TimePeriodsDeleted_ChangesContainOnlyDeletions() + { + var (originalVersion, newVersion, instanceId) = await CreateDataSetInitialAndNextVersion( + nextVersionStatus: DataSetVersionStatus.Mapping, + nextVersionImportStage: Stage.PreviousStage(), + initialVersionMeta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta(), + TimePeriodMetas = + [ + .. DataFixture.DefaultTimePeriodMeta() + .WithCode(TimeIdentifier.CalendarYear) + .WithPeriod("2020") + .Generate(1), + ..DataFixture.DefaultTimePeriodMeta() + .WithCode(TimeIdentifier.AcademicYear) + .ForIndex(0, s => s.SetPeriod("2019")) + .ForIndex(1, s => s.SetPeriod("2020")) + .ForIndex(2, s => s.SetPeriod("2021")) + .Generate(3) + ] + }, + nextVersionMeta: new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta(), + TimePeriodMetas = DataFixture.DefaultTimePeriodMeta() + .WithCode(TimeIdentifier.CalendarYear) + .WithPeriod("2020") + .Generate(1) + }); + + await CreateChanges(instanceId); + + var actualChanges = await GetDbContext() + .TimePeriodMetaChanges + .AsNoTracking() + .Where(c => c.DataSetVersionId == newVersion.Id) + .OrderBy(c => c.Id) + .ToListAsync(); + + Assert.Equal(3, actualChanges.Count); + Assert.All(actualChanges, c => Assert.Equal(newVersion.Id, c.DataSetVersionId)); + Assert.All(actualChanges, c => Assert.Null(c.CurrentStateId)); + + var oldTimePeriodMetas = originalVersion.TimePeriodMetas + .ToDictionary(m => (m.Code, m.Period)); + + Assert.Equal(oldTimePeriodMetas[(TimeIdentifier.AcademicYear, "2019")].Id, + actualChanges[0].PreviousStateId); + Assert.Equal(oldTimePeriodMetas[(TimeIdentifier.AcademicYear, "2020")].Id, + actualChanges[1].PreviousStateId); + Assert.Equal(oldTimePeriodMetas[(TimeIdentifier.AcademicYear, "2021")].Id, + actualChanges[2].PreviousStateId); + } + } + public class UpdateFileStoragePathTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessCompletionOfNextDataSetVersionImportFunctionTests(fixture) @@ -126,17 +3091,9 @@ public class UpdateFileStoragePathTests( [Fact] public async Task Success_PathUpdated() { - var (initialDataSetVersion, _) = await CreateDataSet( - importStage: DataSetVersionImportStage.Completing, - status: DataSetVersionStatus.Published); - - var defaultNextVersion = initialDataSetVersion.DefaultNextVersion(); - - var (nextDataSetVersion, instanceId) = await CreateDataSetVersionAndImport( - dataSetId: initialDataSetVersion.DataSetId, - importStage: Stage, - versionMajor: defaultNextVersion.Major, - versionMinor: defaultNextVersion.Minor); + var (_, nextDataSetVersion, instanceId) = await CreateDataSetInitialAndNextVersion( + nextVersionStatus: DataSetVersionStatus.Mapping, + nextVersionImportStage: Stage); var dataSetVersionPathResolver = GetRequiredService(); var originalStoragePath = dataSetVersionPathResolver.DirectoryPath(nextDataSetVersion); @@ -146,7 +3103,6 @@ public async Task Success_PathUpdated() nextDataSetVersion.VersionMajor++; - publicDataDbContext.DataSetVersions.Attach(nextDataSetVersion); publicDataDbContext.DataSetVersions.Update(nextDataSetVersion); await publicDataDbContext.SaveChangesAsync(); @@ -157,21 +3113,13 @@ public async Task Success_PathUpdated() Assert.False(Directory.Exists(originalStoragePath)); Assert.True(Directory.Exists(newStoragePath)); } - + [Fact] public async Task Success_PathNotUpdated() { - var (initialDataSetVersion, _) = await CreateDataSet( - importStage: DataSetVersionImportStage.Completing, - status: DataSetVersionStatus.Published); - - var defaultNextVersion = initialDataSetVersion.DefaultNextVersion(); - - var (nextDataSetVersion, instanceId) = await CreateDataSetVersionAndImport( - dataSetId: initialDataSetVersion.DataSetId, - importStage: Stage, - versionMajor: defaultNextVersion.Major, - versionMinor: defaultNextVersion.Minor); + var (_, nextDataSetVersion, instanceId) = await CreateDataSetInitialAndNextVersion( + nextVersionStatus: DataSetVersionStatus.Mapping, + nextVersionImportStage: Stage); var dataSetVersionPathResolver = GetRequiredService(); var originalStoragePath = dataSetVersionPathResolver.DirectoryPath(nextDataSetVersion); @@ -198,7 +3146,7 @@ public class CompleteNextDataSetVersionImportProcessingTests( [Fact] public async Task Success() { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); var dataSetVersionPathResolver = GetRequiredService(); Directory.CreateDirectory(dataSetVersionPathResolver.DirectoryPath(dataSetVersion)); @@ -220,7 +3168,7 @@ public async Task Success() [Fact] public async Task DuckDbFileIsDeleted() { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); // Create empty data set version files for all file paths var dataSetVersionPathResolver = GetRequiredService(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs index 2d0179fbc91..b42bea901ab 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs @@ -135,7 +135,7 @@ public class CompleteInitialDataSetVersionProcessingTests( [Fact] public async Task Success() { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); var dataSetVersionPathResolver = GetRequiredService(); Directory.CreateDirectory(dataSetVersionPathResolver.DirectoryPath(dataSetVersion)); @@ -157,7 +157,7 @@ public async Task Success() [Fact] public async Task DuckDbFileIsDeleted() { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); // Create empty data set version files for all file paths var dataSetVersionPathResolver = GetRequiredService(); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs index 451d8632ea2..fba8459ea1a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs @@ -1490,15 +1490,10 @@ private async Task CompleteProcessing(Guid instanceId) private async Task<(Guid instanceId, DataSetVersion initialVersion, DataSetVersion nextVersion)> CreateNextDataSetVersionAndDataFiles(DataSetVersionImportStage importStage) { - var (initialDataSetVersion, _) = await CreateDataSet( - importStage: DataSetVersionImportStage.Completing, - status: DataSetVersionStatus.Published); - - var (nextDataSetVersion, instanceId) = await CreateDataSetVersionAndImport( - dataSetId: initialDataSetVersion.DataSetId, - importStage: importStage, - versionMajor: 1, - versionMinor: 1); + var (initialDataSetVersion, nextDataSetVersion, instanceId) = + await CreateDataSetInitialAndNextVersion( + nextVersionImportStage: importStage, + nextVersionStatus: DataSetVersionStatus.Processing); SetupCsvDataFilesForDataSetVersion(ProcessorTestData.AbsenceSchool, nextDataSetVersion); return (instanceId, initialDataSetVersion, nextDataSetVersion); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/WriteDataFilesFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/WriteDataFilesFunctionTests.cs index d846126866d..56dd08a7dea 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/WriteDataFilesFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/WriteDataFilesFunctionTests.cs @@ -39,7 +39,7 @@ public class WriteDataTests( [MemberData(nameof(Data))] public async Task Success(ProcessorTestData testData) { - var (dataSetVersion, instanceId) = await CreateDataSet(Stage.PreviousStage()); + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await WriteData(testData, dataSetVersion, instanceId); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Models/DataSetVersionMeta.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Models/DataSetVersionMeta.cs new file mode 100644 index 00000000000..25d425df2e1 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Models/DataSetVersionMeta.cs @@ -0,0 +1,16 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests; + +public record DataSetVersionMeta +{ + public IEnumerable? FilterMetas { get; init; } + + public IEnumerable? LocationMetas { get; init; } + + public GeographicLevelMeta? GeographicLevelMeta { get; init; } + + public IEnumerable? IndicatorMetas { get; init; } + + public IEnumerable? TimePeriodMetas { get; init; } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs index 18777e11703..e9d51bad276 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs @@ -40,7 +40,8 @@ public Task DisposeAsync() return Task.CompletedTask; } - protected void SetupCsvDataFilesForDataSetVersion(ProcessorTestData processorTestData, + protected void SetupCsvDataFilesForDataSetVersion( + ProcessorTestData processorTestData, DataSetVersion dataSetVersion) { var dataSetVersionPathResolver = GetRequiredService(); @@ -58,25 +59,76 @@ protected void SetupCsvDataFilesForDataSetVersion(ProcessorTestData processorTes destFileName: dataSetVersionPathResolver.CsvMetadataPath(dataSetVersion)); } - protected async Task<(DataSetVersion dataSetVersion, Guid instanceId)> CreateDataSet( - DataSetVersionImportStage importStage, - DataSetVersionStatus? status = null, - Guid? releaseFileId = null) + protected async Task<(DataSetVersion initialVersion, DataSetVersion nextVersion, Guid instanceId)> + CreateDataSetInitialAndNextVersion( + DataSetVersionStatus nextVersionStatus, + DataSetVersionImportStage nextVersionImportStage, + DataSetVersionMeta? initialVersionMeta = null, + DataSetVersionMeta? nextVersionMeta = null) { - DataSet dataSet = DataFixture.DefaultDataSet(); + var (initialVersion, _) = await CreateDataSetInitialVersion( + dataSetStatus: DataSetStatus.Published, + dataSetVersionStatus: DataSetVersionStatus.Published, + importStage: DataSetVersionImportStage.Completing, + meta: initialVersionMeta); + + var (nextVersion, instanceId) = await CreateDataSetNextVersion( + initialVersion: initialVersion, + status: nextVersionStatus, + importStage: nextVersionImportStage, + meta: nextVersionMeta); + + return (initialVersion, nextVersion, instanceId); + } + + protected async Task<(DataSetVersion dataSetVersion, Guid instanceId)> + CreateDataSetInitialVersion( + DataSetVersionImportStage importStage, + DataSetStatus dataSetStatus = DataSetStatus.Draft, + DataSetVersionStatus dataSetVersionStatus = DataSetVersionStatus.Processing, + Guid? releaseFileId = null, + DataSetVersionMeta? meta = null) + { + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatus(dataSetStatus); await AddTestData(context => context.DataSets.Add(dataSet)); - return await CreateDataSetVersionAndImport(dataSet.Id, importStage, status, releaseFileId); + return await CreateDataSetVersion( + dataSetId: dataSet.Id, + importStage: importStage, + status: dataSetVersionStatus, + releaseFileId: releaseFileId, + meta: meta); } - protected async Task<(DataSetVersion dataSetVersion, Guid instanceId)> CreateDataSetVersionAndImport( + protected async Task<(DataSetVersion nextVersion, Guid instanceId)> + CreateDataSetNextVersion( + DataSetVersion initialVersion, + DataSetVersionStatus status, + DataSetVersionImportStage importStage, + DataSetVersionMeta? meta = null) + { + var defaultNextVersion = initialVersion.DefaultNextVersion(); + + return await CreateDataSetVersion( + dataSetId: initialVersion.DataSetId, + status: status, + importStage: importStage, + versionMajor: defaultNextVersion.Major, + versionMinor: defaultNextVersion.Minor, + meta: meta); + } + + private async Task<(DataSetVersion dataSetVersion, Guid instanceId)> CreateDataSetVersion( Guid dataSetId, DataSetVersionImportStage importStage, - DataSetVersionStatus? status = null, + DataSetVersionStatus status = DataSetVersionStatus.Processing, Guid? releaseFileId = null, int versionMajor = 1, - int versionMinor = 0) + int versionMinor = 0, + DataSetVersionMeta? meta = null) { await using var publicDataDbContext = GetDbContext(); @@ -88,10 +140,10 @@ protected void SetupCsvDataFilesForDataSetVersion(ProcessorTestData processorTes .DefaultDataSetVersionImport() .WithStage(importStage); - DataSetVersion dataSetVersion = DataFixture + var dataSetVersionGenerator = DataFixture .DefaultDataSetVersion() .WithDataSet(dataSet) - .WithStatus(status ?? DataSetVersionStatus.Processing) + .WithStatus(status) .WithImports(() => [dataSetVersionImport]) .WithVersionNumber(major: versionMajor, minor: versionMinor) .FinishWith(dsv => @@ -111,6 +163,38 @@ protected void SetupCsvDataFilesForDataSetVersion(ProcessorTestData processorTes } }); + if (meta?.FilterMetas is not null) + { + dataSetVersionGenerator = dataSetVersionGenerator + .WithFilterMetas(() => meta.FilterMetas); + } + + if (meta?.LocationMetas is not null) + { + dataSetVersionGenerator = dataSetVersionGenerator + .WithLocationMetas(() => meta.LocationMetas); + } + + if (meta?.GeographicLevelMeta is not null) + { + dataSetVersionGenerator = dataSetVersionGenerator + .WithGeographicLevelMeta(() => meta.GeographicLevelMeta); + } + + if (meta?.IndicatorMetas is not null) + { + dataSetVersionGenerator = dataSetVersionGenerator + .WithIndicatorMetas(() => meta.IndicatorMetas); + } + + if (meta?.TimePeriodMetas is not null) + { + dataSetVersionGenerator = dataSetVersionGenerator + .WithTimePeriodMetas(() => meta.TimePeriodMetas); + } + + var dataSetVersion = dataSetVersionGenerator.Generate(); + await AddTestData(context => { context.DataSetVersions.Add(dataSetVersion); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs index d52df5a22a2..b990d7e2b45 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs @@ -16,6 +16,8 @@ internal static class ActivityNames public const string CompleteNextDataSetVersionMappingProcessing = nameof(ProcessNextDataSetVersionMappingsFunction.CompleteNextDataSetVersionMappingProcessing); + public const string CreateChanges = + nameof(ProcessCompletionOfNextDataSetVersionFunction.CreateChanges); public const string UpdateFileStoragePath = nameof(ProcessCompletionOfNextDataSetVersionFunction.UpdateFileStoragePath); public const string CompleteNextDataSetVersionImportProcessing = diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunction.cs index 9918adf75c6..a71de9a718c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunction.cs @@ -2,6 +2,7 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using Microsoft.Azure.Functions.Worker; using Microsoft.DurableTask; @@ -12,7 +13,8 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Funct public class ProcessCompletionOfNextDataSetVersionFunction( PublicDataDbContext publicDataDbContext, - IDataSetVersionPathResolver dataSetVersionPathResolver) : BaseProcessDataSetVersionFunction(publicDataDbContext) + IDataSetVersionPathResolver dataSetVersionPathResolver, + IDataSetVersionChangeService dataSetVersionChangeService) : BaseProcessDataSetVersionFunction(publicDataDbContext) { [Function(nameof(ProcessCompletionOfNextDataSetVersion))] public async Task ProcessCompletionOfNextDataSetVersion( @@ -31,6 +33,7 @@ public async Task ProcessCompletionOfNextDataSetVersion( { await context.CallActivity(ActivityNames.UpdateFileStoragePath, logger, context.InstanceId); await context.CallActivity(ActivityNames.ImportMetadata, logger, context.InstanceId); + await context.CallActivity(ActivityNames.CreateChanges, logger, context.InstanceId); await context.CallActivity(ActivityNames.ImportData, logger, context.InstanceId); await context.CallActivity(ActivityNames.WriteDataFiles, logger, context.InstanceId); await context.CallActivity(ActivityNames.CompleteNextDataSetVersionImportProcessing, logger, @@ -47,6 +50,16 @@ await context.CallActivity(ActivityNames.CompleteNextDataSetVersionImportProcess } } + [Function(ActivityNames.CreateChanges)] + public async Task CreateChanges( + [ActivityTrigger] Guid instanceId, + CancellationToken cancellationToken) + { + var dataSetVersionImport = await GetDataSetVersionImport(instanceId, cancellationToken); + await UpdateImportStage(dataSetVersionImport, DataSetVersionImportStage.CreatingChanges, cancellationToken); + await dataSetVersionChangeService.CreateChanges(dataSetVersionImport.DataSetVersionId, cancellationToken); + } + [Function(ActivityNames.UpdateFileStoragePath)] public async Task UpdateFileStoragePath( [ActivityTrigger] Guid instanceId, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs index b5ce101e6cc..e1f8607b2dc 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/ProcessorHostBuilder.cs @@ -77,6 +77,7 @@ public static IHostBuilder ConfigureProcessorHostBuilder(this IHostBuilder hostB .AddScoped() .AddScoped() .AddScoped() + .AddScoped() .AddScoped() .AddScoped() .AddScoped() diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionChangeService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionChangeService.cs new file mode 100644 index 00000000000..54afe6f61b3 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionChangeService.cs @@ -0,0 +1,507 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using GovUk.Education.ExploreEducationStatistics.Common.Model.Data; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; +using Microsoft.EntityFrameworkCore; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services; + +internal class DataSetVersionChangeService(PublicDataDbContext publicDataDbContext) + : IDataSetVersionChangeService +{ + public async Task CreateChanges( + Guid nextDataSetVersionId, + CancellationToken cancellationToken = default) + { + var nextVersion = await publicDataDbContext + .DataSetVersions + .Include(dsv => dsv.DataSet) + .SingleAsync(dsv => dsv.Id == nextDataSetVersionId, cancellationToken); + + var previousVersionId = nextVersion.DataSet.LatestLiveVersionId!.Value; + + await publicDataDbContext.RequireTransaction(async () => + { + await CreateFilterChanges( + previousVersionId: previousVersionId, + nextVersionId: nextVersion.Id, + cancellationToken: cancellationToken); + + await CreateLocationChanges( + previousVersionId: previousVersionId, + nextVersionId: nextVersion.Id, + cancellationToken: cancellationToken); + + await CreateGeographicLevelChange( + previousVersionId: previousVersionId, + nextVersionId: nextVersion.Id, + cancellationToken: cancellationToken); + + await CreateIndicatorChanges( + previousVersionId: previousVersionId, + nextVersionId: nextVersion.Id, + cancellationToken: cancellationToken); + + await CreateTimePeriodChanges( + previousVersionId: previousVersionId, + nextVersionId: nextVersion.Id, + cancellationToken: cancellationToken); + }); + } + + private async Task CreateFilterChanges( + Guid previousVersionId, + Guid nextVersionId, + CancellationToken cancellationToken) + { + var oldMetas = await GetFilterMetas(previousVersionId, cancellationToken); + var newMetas = await GetFilterMetas(nextVersionId, cancellationToken); + + var metaDeletions = new List(); + var optionMetaDeletions = new List(); + var metaUpdates = new List(); + var optionMetaUpdates = new List(); + var metaAdditions = new List(); + var optionMetaAdditions = new List(); + + foreach (var (filterPublicId, oldFilterTuple) in oldMetas) + { + if (newMetas.TryGetValue(filterPublicId, out var newFilterTuple)) + { + // Filter updated + if (!IsFilterEqual(newFilterTuple.FilterMeta, oldFilterTuple.FilterMeta)) + { + metaUpdates.Add(new FilterMetaChange + { + DataSetVersionId = nextVersionId, + PreviousState = oldFilterTuple.FilterMeta, + PreviousStateId = oldFilterTuple.FilterMeta.Id, + CurrentState = newFilterTuple.FilterMeta, + CurrentStateId = newFilterTuple.FilterMeta.Id + }); + } + + foreach (var (optionPublicId, oldOptionLink) in oldFilterTuple.OptionLinks) + { + if (!newFilterTuple.OptionLinks.TryGetValue(optionPublicId, out var newOptionLink)) + { + // Filter option deleted + optionMetaDeletions.Add(new FilterOptionMetaChange + { + DataSetVersionId = nextVersionId, + PreviousState = FilterOptionMetaChange.State.Create(oldOptionLink) + }); + } + // Filter option updated + else if (!IsFilterOptionEqual(newOptionLink.Option, oldOptionLink.Option)) + { + optionMetaUpdates.Add(new FilterOptionMetaChange + { + DataSetVersionId = nextVersionId, + PreviousState = FilterOptionMetaChange.State.Create(oldOptionLink), + CurrentState = FilterOptionMetaChange.State.Create(newOptionLink) + }); + + // Improving performance by removing from the new meta to avoid having to loop + // over it again when finding the additions + newFilterTuple.OptionLinks.Remove(optionPublicId); + } + } + } + else + { + // Filter deleted + metaDeletions.Add(new FilterMetaChange + { + DataSetVersionId = nextVersionId, + PreviousState = oldFilterTuple.FilterMeta, + PreviousStateId = oldFilterTuple.FilterMeta.Id, + }); + } + } + + foreach (var (filterPublicId, newFilterTuple) in newMetas) + { + if (oldMetas.TryGetValue(filterPublicId, out var oldFilterTuple)) + { + foreach (var (optionPublicId, newOptionLink) in newFilterTuple.OptionLinks) + { + if (!oldFilterTuple.OptionLinks.ContainsKey(optionPublicId)) + { + // Filter option added + optionMetaAdditions.Add(new FilterOptionMetaChange + { + DataSetVersionId = nextVersionId, + CurrentState = FilterOptionMetaChange.State.Create(newOptionLink) + }); + } + } + } + else + { + // Filter added + metaAdditions.Add(new FilterMetaChange + { + DataSetVersionId = nextVersionId, + CurrentState = newFilterTuple.FilterMeta, + CurrentStateId = newFilterTuple.FilterMeta.Id + }); + } + } + + // The meta changes are inserted into the DB in the following order: + // - Deletions + // - Updates + // - Additions + publicDataDbContext.FilterMetaChanges.AddRange( + [ + .. metaDeletions.NaturalOrderBy(c => c.PreviousState!.Label), + .. metaUpdates.NaturalOrderBy(c => c.PreviousState!.Label), + .. metaAdditions.NaturalOrderBy(c => c.CurrentState!.Label) + ]); + + publicDataDbContext.FilterOptionMetaChanges.AddRange( + [ + .. optionMetaDeletions.NaturalOrderBy(c => c.PreviousState!.Option.Label), + .. optionMetaUpdates.NaturalOrderBy(c => c.PreviousState!.Option.Label), + .. optionMetaAdditions.NaturalOrderBy(c => c.CurrentState!.Option.Label) + ]); + + await publicDataDbContext.SaveChangesAsync(cancellationToken); + } + + private static bool IsFilterEqual(FilterMeta filterMeta1, FilterMeta filterMeta2) + { + return filterMeta1.Column == filterMeta2.Column + && filterMeta1.Label == filterMeta2.Label + && filterMeta1.Hint == filterMeta2.Hint; + } + + private static bool IsFilterOptionEqual(FilterOptionMeta filterOptionMeta1, FilterOptionMeta filterOptionMeta2) + { + return filterOptionMeta1.Label == filterOptionMeta2.Label; + } + + private async Task CreateLocationChanges( + Guid previousVersionId, + Guid nextVersionId, + CancellationToken cancellationToken) + { + var oldMetas = await GetLocationMetas(previousVersionId, cancellationToken); + var newMetas = await GetLocationMetas(nextVersionId, cancellationToken); + + var metaDeletions = new List(); + var optionMetaDeletions = new List(); + var optionMetaUpdates = new List(); + var metaAdditions = new List(); + var optionMetaAdditions = new List(); + + foreach (var (locationLevel, oldLocationTuple) in oldMetas) + { + if (newMetas.TryGetValue(locationLevel, out var newLocationTuple)) + { + foreach (var (optionPublicId, oldOptionLink) in oldLocationTuple.OptionLinks) + { + if (!newLocationTuple.OptionLinks.TryGetValue(optionPublicId, out var newOptionLink)) + { + // Location option deleted + optionMetaDeletions.Add(new LocationOptionMetaChange + { + DataSetVersionId = nextVersionId, + PreviousState = LocationOptionMetaChange.State.Create(oldOptionLink) + }); + } + // Location option updated + else if (!IsLocationOptionEqual(newOptionLink.Option, oldOptionLink.Option)) + { + optionMetaUpdates.Add(new LocationOptionMetaChange + { + DataSetVersionId = nextVersionId, + PreviousState = LocationOptionMetaChange.State.Create(oldOptionLink), + CurrentState = LocationOptionMetaChange.State.Create(newOptionLink) + }); + + // Improving performance by removing from the new meta to avoid having to loop + // over it again when finding the additions + newLocationTuple.OptionLinks.Remove(optionPublicId); + } + } + } + else + { + // Location deleted + metaDeletions.Add(new LocationMetaChange + { + DataSetVersionId = nextVersionId, + PreviousState = oldLocationTuple.LocationMeta, + PreviousStateId = oldLocationTuple.LocationMeta.Id, + }); + } + } + + foreach (var (locationLevel, newLocationTuple) in newMetas) + { + if (oldMetas.TryGetValue(locationLevel, out var oldLocationTuple)) + { + foreach (var (optionPublicId, newOptionLink) in newLocationTuple.OptionLinks) + { + if (!oldLocationTuple.OptionLinks.ContainsKey(optionPublicId)) + { + // Location option added + optionMetaAdditions.Add(new LocationOptionMetaChange + { + DataSetVersionId = nextVersionId, + CurrentState = LocationOptionMetaChange.State.Create(newOptionLink) + }); + } + } + } + else + { + // Location added + metaAdditions.Add(new LocationMetaChange + { + DataSetVersionId = nextVersionId, + CurrentState = newLocationTuple.LocationMeta, + CurrentStateId = newLocationTuple.LocationMeta.Id + }); + } + } + + // The meta changes are inserted into the DB in the following order: + // - Deletions + // - Updates + // - Additions + publicDataDbContext.LocationMetaChanges.AddRange( + [ + .. metaDeletions.NaturalOrderBy(c => c.PreviousState!.Level.GetEnumLabel()), + .. metaAdditions.NaturalOrderBy(c => c.CurrentState!.Level.GetEnumLabel()) + ]); + + publicDataDbContext.LocationOptionMetaChanges.AddRange( + [ + .. optionMetaDeletions.NaturalOrderBy(c => c.PreviousState!.Option.Label), + .. optionMetaUpdates.NaturalOrderBy(c => c.PreviousState!.Option.Label), + .. optionMetaAdditions.NaturalOrderBy(c => c.CurrentState!.Option.Label) + ]); + + await publicDataDbContext.SaveChangesAsync(cancellationToken); + } + + private static bool IsLocationOptionEqual(LocationOptionMeta locationOptionMeta1, LocationOptionMeta locationOptionMeta2) + { + // We need to check if the label, and any codes have changed. + // However, the LocationOptionMeta table is constrained so that every row is unique and we only create new options + // when something has changed. This means we can get away with just checking that the ids are equal. + return locationOptionMeta1.Id == locationOptionMeta2.Id; + } + + private async Task CreateGeographicLevelChange( + Guid previousVersionId, + Guid nextVersionId, + CancellationToken cancellationToken) + { + var oldGeographicLevelMeta = await GetGeographicLevelMeta(previousVersionId, cancellationToken); + var newGeographicLevelMeta = await GetGeographicLevelMeta(nextVersionId, cancellationToken); + + var levelsAreEqual = + newGeographicLevelMeta.Levels.Order().SequenceEqual(oldGeographicLevelMeta.Levels.Order()); + + if (!levelsAreEqual) + { + publicDataDbContext.GeographicLevelMetaChanges.Add(new GeographicLevelMetaChange + { + DataSetVersionId = nextVersionId, + PreviousStateId = oldGeographicLevelMeta.Id, + CurrentStateId = newGeographicLevelMeta.Id + }); + await publicDataDbContext.SaveChangesAsync(cancellationToken); + } + } + + private async Task CreateIndicatorChanges( + Guid previousVersionId, + Guid nextVersionId, + CancellationToken cancellationToken) + { + var oldMetas = await GetIndicatorMetas(previousVersionId, cancellationToken); + var newMetas = await GetIndicatorMetas(nextVersionId, cancellationToken); + + var metaDeletions = new List(); + var metaUpdates = new List(); + var metaAdditions = new List(); + + foreach (var (indicatorPublicId, oldIndicator) in oldMetas) + { + if (newMetas.TryGetValue(indicatorPublicId, out var newIndicator)) + { + // Indicator updated + if (!IsIndicatorEqual(newIndicator, oldIndicator)) + { + metaUpdates.Add(new IndicatorMetaChange + { + DataSetVersionId = nextVersionId, + PreviousState = oldIndicator, + PreviousStateId = oldIndicator.Id, + CurrentState = newIndicator, + CurrentStateId = newIndicator.Id + }); + } + } + else + { + // Indicator deleted + metaDeletions.Add(new IndicatorMetaChange + { + DataSetVersionId = nextVersionId, + PreviousState = oldIndicator, + PreviousStateId = oldIndicator.Id, + }); + } + } + + foreach (var (indicatorPublicId, newIndicator) in newMetas) + { + if (!oldMetas.TryGetValue(indicatorPublicId, out var oldIndicator)) + { + // Indicator added + metaAdditions.Add(new IndicatorMetaChange + { + DataSetVersionId = nextVersionId, + CurrentState = newIndicator, + CurrentStateId = newIndicator.Id + }); + } + } + + // The meta changes are inserted into the DB in the following order: + // - Deletions + // - Updates + // - Additions + publicDataDbContext.IndicatorMetaChanges.AddRange( + [ + .. metaDeletions.NaturalOrderBy(c => c.PreviousState!.Label), + .. metaUpdates.NaturalOrderBy(c => c.PreviousState!.Label), + .. metaAdditions.NaturalOrderBy(c => c.CurrentState!.Label) + ]); + + await publicDataDbContext.SaveChangesAsync(cancellationToken); + } + + private static bool IsIndicatorEqual(IndicatorMeta indicatorMeta1, IndicatorMeta indicatorMeta2) + { + return indicatorMeta1.Column == indicatorMeta2.Column + && indicatorMeta1.Label == indicatorMeta2.Label + && indicatorMeta1.Unit == indicatorMeta2.Unit + && indicatorMeta1.DecimalPlaces == indicatorMeta2.DecimalPlaces; + } + + private async Task CreateTimePeriodChanges( + Guid previousVersionId, + Guid nextVersionId, + CancellationToken cancellationToken) + { + var oldTimePeriodMetas = await GetTimePeriodMetas(previousVersionId, cancellationToken); + var newTimePeriodMetas = await GetTimePeriodMetas(nextVersionId, cancellationToken); + + var timePeriodMetaDeletions = oldTimePeriodMetas + .Except(newTimePeriodMetas, TimePeriodMeta.CodePeriodComparer) + .OrderBy(timePeriodMeta => timePeriodMeta.Period) + .ThenBy(timePeriodMeta => timePeriodMeta.Code) + .Select(timePeriodMeta => new TimePeriodMetaChange + { + DataSetVersionId = nextVersionId, + PreviousStateId = timePeriodMeta.Id, + CurrentStateId = null + }) + .ToList(); + + var timePeriodMetaAdditions = newTimePeriodMetas + .Except(oldTimePeriodMetas, TimePeriodMeta.CodePeriodComparer) + .OrderBy(timePeriodMeta => timePeriodMeta.Period) + .ThenBy(timePeriodMeta => timePeriodMeta.Code) + .Select(timePeriodMeta => new TimePeriodMetaChange + { + DataSetVersionId = nextVersionId, + PreviousStateId = null, + CurrentStateId = timePeriodMeta.Id + }) + .ToList(); + + publicDataDbContext.TimePeriodMetaChanges.AddRange([.. timePeriodMetaDeletions, .. timePeriodMetaAdditions]); + await publicDataDbContext.SaveChangesAsync(cancellationToken); + } + + private async + Task OptionLinks)>> + GetFilterMetas( + Guid dataSetVersionId, + CancellationToken cancellationToken) + { + return await publicDataDbContext + .FilterMetas + .Include(m => m.OptionLinks) + .ThenInclude(l => l.Option) + .Where(m => m.DataSetVersionId == dataSetVersionId) + .ToDictionaryAsync( + m => m.PublicId, + m => ( + FilterMeta: m, + OptionLinks: m.OptionLinks.ToDictionary(l => l.PublicId) + ), + cancellationToken); + } + + private async Task GetGeographicLevelMeta( + Guid dataSetVersionId, + CancellationToken cancellationToken) + { + return await publicDataDbContext + .GeographicLevelMetas + .AsNoTracking() + .SingleAsync(m => m.DataSetVersionId == dataSetVersionId, cancellationToken); + } + + private async Task> GetIndicatorMetas( + Guid dataSetVersionId, + CancellationToken cancellationToken) + { + return await publicDataDbContext + .IndicatorMetas + .Where(m => m.DataSetVersionId == dataSetVersionId) + .ToDictionaryAsync( + i => i.PublicId, + cancellationToken); + } + + private async Task OptionLinks)>> GetLocationMetas( + Guid previousVersionId, CancellationToken cancellationToken) + { + return await publicDataDbContext + .LocationMetas + .Include(lm => lm.OptionLinks) + .ThenInclude(oml => oml.Option) + .Where(lm => lm.DataSetVersionId == previousVersionId) + .ToDictionaryAsync( + lm => lm.Level, + lm => + ( + LocationMeta: lm, + OptionLinks: lm.OptionLinks.ToDictionary(loml => loml.PublicId) + ), + cancellationToken); + } + + private async Task> GetTimePeriodMetas( + Guid dataSetVersionId, + CancellationToken cancellationToken) + { + return await publicDataDbContext + .TimePeriodMetas + .AsNoTracking() + .Where(tpm => tpm.DataSetVersionId == dataSetVersionId) + .ToListAsync(cancellationToken); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetVersionChangeService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetVersionChangeService.cs new file mode 100644 index 00000000000..22a72cf3a89 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetVersionChangeService.cs @@ -0,0 +1,8 @@ +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; + +public interface IDataSetVersionChangeService +{ + Task CreateChanges( + Guid nextDataSetVersionId, + CancellationToken cancellationToken = default); +} From fd42dcfd32ee5f4992ee041691ba2a7154aa6a8f Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 7 Oct 2024 14:17:10 +0100 Subject: [PATCH 30/80] EES-5526 - removed potentially redundant timezone code --- .../ReleaseApiDataSetPreviewTokenPage.tsx | 20 ++++++------ .../tests/admin/bau/prerelease.robot | 15 ++++----- .../admin/bau/prerelease_and_amend.robot | 9 +++--- tests/robot-tests/tests/libs/utilities.py | 32 +++++++++---------- 4 files changed, 35 insertions(+), 41 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx index 19c46a479ce..7d733c9cc10 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/ReleaseApiDataSetPreviewTokenPage.tsx @@ -1,5 +1,5 @@ import Link from '@admin/components/Link'; -import {useConfig} from '@admin/contexts/ConfigContext'; +import { useConfig } from '@admin/contexts/ConfigContext'; import { releaseApiDataSetDetailsRoute, releaseApiDataSetPreviewRoute, @@ -10,7 +10,7 @@ import { import previewTokenQueries from '@admin/queries/previewTokenQueries'; import apiDataSetQueries from '@admin/queries/apiDataSetQueries'; import previewTokenService from '@admin/services/previewTokenService'; -import {useLastLocation} from '@admin/contexts/LastLocationContext'; +import { useLastLocation } from '@admin/contexts/LastLocationContext'; import CodeBlock from '@common/components/CodeBlock'; import LoadingSpinner from '@common/components/LoadingSpinner'; import Button from '@common/components/Button'; @@ -20,24 +20,24 @@ import ModalConfirm from '@common/components/ModalConfirm'; import Tabs from '@common/components/Tabs'; import TabsSection from '@common/components/TabsSection'; import ApiDataSetQuickStart from '@common/modules/data-catalogue/components/ApiDataSetQuickStart'; -import {useQuery} from '@tanstack/react-query'; -import {generatePath, useHistory, useParams} from 'react-router-dom'; +import { useQuery } from '@tanstack/react-query'; +import { generatePath, useHistory, useParams } from 'react-router-dom'; import React from 'react'; export default function ReleaseApiDataSetPreviewTokenPage() { const history = useHistory(); const lastLocation = useLastLocation(); - const {publicApiUrl, publicApiDocsUrl} = useConfig(); + const { publicApiUrl, publicApiDocsUrl } = useConfig(); - const {dataSetId, previewTokenId, releaseId, publicationId} = + const { dataSetId, previewTokenId, releaseId, publicationId } = useParams(); - const {data: dataSet, isLoading: isLoadingDataSet} = useQuery( + const { data: dataSet, isLoading: isLoadingDataSet } = useQuery( apiDataSetQueries.get(dataSetId), ); - const {data: previewToken, isLoading: isLoadingPreviewTokenId} = useQuery({ + const { data: previewToken, isLoading: isLoadingPreviewTokenId } = useQuery({ ...previewTokenQueries.get(previewTokenId), }); @@ -120,8 +120,8 @@ export default function ReleaseApiDataSetPreviewTokenPage() { {previewToken.expiry} - - {' '}(local time) + {' '} + (local time)

diff --git a/tests/robot-tests/tests/admin/bau/prerelease.robot b/tests/robot-tests/tests/admin/bau/prerelease.robot index 6da64dc4cac..70bf790c00e 100644 --- a/tests/robot-tests/tests/admin/bau/prerelease.robot +++ b/tests/robot-tests/tests/admin/bau/prerelease.robot @@ -29,7 +29,8 @@ Upload subject user navigates to draft release page from dashboard ${PUBLICATION_NAME} ... Calendar year 2000 - user uploads subject and waits until complete UI test subject upload-file-test.csv upload-file-test.meta.csv + user uploads subject and waits until complete UI test subject upload-file-test.csv + ... upload-file-test.meta.csv Add metadata guidance user clicks link Data guidance @@ -141,11 +142,9 @@ Validate prerelease has not started user checks nth breadcrumb contains 1 Home user checks nth breadcrumb contains 2 Pre-release access - ${tomorrow}= get current datetime %Y-%m-%dT00:00:00 1 Europe/London - ${day_after_tomorrow}= get current datetime %Y-%m-%dT%H:%M:%S 2 Europe/London + ${time_start}= get current london datetime %-d %B %Y at 00:00 1 + ${time_end}= get current london datetime %-d %B %Y 2 - ${time_start}= format uk to local datetime ${tomorrow} %-d %B %Y at %H:%M - ${time_end}= format uk to local datetime ${day_after_tomorrow} %-d %B %Y user checks page contains ... Pre-release access will be available from ${time_start} until it is published on ${time_end}. @@ -248,11 +247,9 @@ Validate prerelease has not started for Analyst user user checks nth breadcrumb contains 1 Home user checks nth breadcrumb contains 2 Pre-release access - ${tomorrow}= get current datetime %Y-%m-%dT00:00:00 1 - ${day_after_tomorrow}= get current datetime %Y-%m-%dT%H:%M:%S 2 + ${time_start}= get current london datetime %-d %B %Y at 00:00 1 + ${time_end}= get current london datetime %-d %B %Y 2 - ${time_start}= format uk to local datetime ${tomorrow} %-d %B %Y at %H:%M - ${time_end}= format uk to local datetime ${day_after_tomorrow} %-d %B %Y user checks page contains ... Pre-release access will be available from ${time_start} until it is published on ${time_end}. diff --git a/tests/robot-tests/tests/admin/bau/prerelease_and_amend.robot b/tests/robot-tests/tests/admin/bau/prerelease_and_amend.robot index 64b350d69ea..92df2d946e8 100644 --- a/tests/robot-tests/tests/admin/bau/prerelease_and_amend.robot +++ b/tests/robot-tests/tests/admin/bau/prerelease_and_amend.robot @@ -30,7 +30,8 @@ Upload subject user navigates to draft release page from dashboard ${PUBLICATION_NAME} ... Calendar year 2000 - user uploads subject and waits until complete UI test subject upload-file-test.csv upload-file-test.meta.csv + user uploads subject and waits until complete UI test subject upload-file-test.csv + ... upload-file-test.meta.csv Add metadata guidance user clicks link Data guidance @@ -238,11 +239,9 @@ Validate prerelease window is not yet open for Analyst user user checks nth breadcrumb contains 1 Home user checks nth breadcrumb contains 2 Pre-release access - ${tomorrow}= get current datetime %Y-%m-%dT00:00:00 1 - ${day_after_tomorrow}= get current datetime %Y-%m-%dT%H:%M:%S 2 + ${time_start}= get current london datetime %-d %B %Y at 00:00 1 + ${time_end}= get current london datetime %-d %B %Y 2 - ${time_start}= format uk to local datetime ${tomorrow} %-d %B %Y at %H:%M - ${time_end}= format uk to local datetime ${day_after_tomorrow} %-d %B %Y user checks page contains ... Pre-release access will be available from ${time_start} until it is published on ${time_end}. diff --git a/tests/robot-tests/tests/libs/utilities.py b/tests/robot-tests/tests/libs/utilities.py index 6b1d449da3d..ea2782e19eb 100644 --- a/tests/robot-tests/tests/libs/utilities.py +++ b/tests/robot-tests/tests/libs/utilities.py @@ -3,20 +3,20 @@ import json import os import re +import time from typing import Union +from urllib.parse import urlparse, urlunparse -import time import pytz import utilities_init import visual from robot.libraries.BuiltIn import BuiltIn +from selenium.common.exceptions import NoSuchElementException from selenium.webdriver.common.by import By from selenium.webdriver.remote.webelement import WebElement -from selenium.common.exceptions import NoSuchElementException from SeleniumLibrary.utils import is_noney from tests.libs.logger import get_logger from tests.libs.selenium_elements import element_finder, sl, waiting -from urllib.parse import urlparse, urlunparse logger = get_logger(__name__) @@ -99,8 +99,13 @@ def retry_or_fail_with_delay(func, retries=5, delay=1.0, *args, **kwargs): def user_waits_until_parent_contains_element( - parent_locator: object, child_locator: str, timeout: int = None, error: str = None, count: int = None, - retries: int = 5, delay: float = 1.0 + parent_locator: object, + child_locator: str, + timeout: int = None, + error: str = None, + count: int = None, + retries: int = 5, + delay: float = 1.0, ): try: child_locator = _normalise_child_locator(child_locator) @@ -274,21 +279,12 @@ def set_cookie_from_json(cookie_json): sl().driver.add_cookie(cookie_dict) -def format_uk_to_local_datetime(uk_local_datetime: str, strf: str) -> str: - if os.name == "nt": - strf = strf.replace("%-", "%#") - - tz = pytz.timezone("Europe/London") - - return tz.localize(datetime.datetime.fromisoformat(uk_local_datetime)).strftime(strf) - - def get_current_datetime(strf: str, offset_days: int = 0, timezone: str = "UTC") -> str: return format_datetime(datetime.datetime.now(pytz.timezone(timezone)) + datetime.timedelta(days=offset_days), strf) def get_current_london_datetime(strf: str, offset_days: int = 0) -> str: - return get_current_datetime(strf, offset_days, 'Europe/London') + return get_current_datetime(strf, offset_days, "Europe/London") def get_current_local_datetime(strf: str, offset_days: int = 0) -> str: @@ -448,7 +444,8 @@ def remove_auth_from_url(publicUrl: str): netloc += f":{parsed_url.port}" modified_url_without_auth = urlunparse( - (parsed_url.scheme, netloc, parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment)) + (parsed_url.scheme, netloc, parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment) + ) return modified_url_without_auth @@ -463,5 +460,6 @@ def get_child_element_with_retry(parent_locator: object, child_locator: str, max time.sleep(retry_delay) raise AssertionError(f"Failed to find child element after {max_retries} retries.") + def _get_browser_timezone(): - return sl().driver.execute_script('return Intl.DateTimeFormat().resolvedOptions().timeZone;') + return sl().driver.execute_script("return Intl.DateTimeFormat().resolvedOptions().timeZone;") From 3d383e756461daada3fce1e57fac9967c69a3f9e Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 1 Oct 2024 16:58:04 +0100 Subject: [PATCH 31/80] EES-5540 - fixed bug with Pabot and the Robot --rerunfailedsuites flag, which caused multiple same failed suites to run in parallel on rerun. --- tests/robot-tests/.env.example | 1 - tests/robot-tests/args_and_variables.py | 1 - tests/robot-tests/reports.py | 109 +++++++++++ tests/robot-tests/run_tests.py | 179 ++++++------------ tests/robot-tests/test_runners.py | 110 +++++++++++ .../data_set_page_absence_in_prus.robot | 2 +- .../libs/admin/manage-content-common.robot | 4 +- tests/robot-tests/tests/libs/common.robot | 23 +-- tests/robot-tests/tests/libs/fail_fast.py | 59 ++++-- tests/robot-tests/tests/libs/slack.py | 16 +- tests/robot-tests/tests/libs/utilities.py | 37 +--- tests/robot-tests/tests/libs/visual.py | 4 +- .../public_api_cancel_and_removal.robot | 4 +- .../public_api/public_api_preview_token.robot | 4 +- .../public_api_resolve_mapping_statuses.robot | 2 +- .../public_api/public_api_restricted.robot | 6 +- 16 files changed, 345 insertions(+), 216 deletions(-) create mode 100644 tests/robot-tests/reports.py create mode 100644 tests/robot-tests/test_runners.py diff --git a/tests/robot-tests/.env.example b/tests/robot-tests/.env.example index 75ca5508004..2f4b139d900 100644 --- a/tests/robot-tests/.env.example +++ b/tests/robot-tests/.env.example @@ -22,7 +22,6 @@ WAIT_LONG=180 # Variable used throughout tests (in seconds) WAIT_DATA_FILE_IMPORT=180 # (int) Time to wait for a data file to import TIMEOUT=30 # Variable used throughout tests (in seconds) IMPLICIT_WAIT=15 # Variable used throughout tests (in seconds) -FAIL_TEST_SUITES_FAST=1 # ("0" or "1") Fails entire test suite after any failure IDENTITY_PROVIDER= # ("AZURE" or "KEYCLOAK") opt into using either a Keycloak or Azure based identity provider PUBLISHER_FUNCTIONS_URL=http://localhost:7072 # The base URL for the Publisher Functions project for the environment SLACK_APP_TOKEN= # The "Bot User OAuth Token" used by the Slack app to send channel alerts diff --git a/tests/robot-tests/args_and_variables.py b/tests/robot-tests/args_and_variables.py index 99041703501..994d8bec279 100644 --- a/tests/robot-tests/args_and_variables.py +++ b/tests/robot-tests/args_and_variables.py @@ -183,7 +183,6 @@ def validate_environment_variables(): "WAIT_LONG", "WAIT_SMALL", "WAIT_DATA_FILE_IMPORT", - "FAIL_TEST_SUITES_FAST", "IDENTITY_PROVIDER", "WAIT_CACHE_EXPIRY", "ADMIN_EMAIL", diff --git a/tests/robot-tests/reports.py b/tests/robot-tests/reports.py new file mode 100644 index 00000000000..10f89095f69 --- /dev/null +++ b/tests/robot-tests/reports.py @@ -0,0 +1,109 @@ +import os +import glob +import shutil +from bs4 import BeautifulSoup +from robot import rebot_cli as robot_rebot_cli +from tests.libs.logger import get_logger + +main_results_folder = "test-results" + +logger = get_logger(__name__) + + +# Merge multiple Robot test reports and assets together into the main test results folder. +def merge_robot_reports(number_of_test_runs: int): + + run_1_folder=f"{main_results_folder}{os.sep}run-1" + + for file in os.listdir(run_1_folder): + _copy_to_destination_folder(run_1_folder, file, main_results_folder) + + for test_run in range(2, number_of_test_runs + 1): + + logger.info(f"Merging test run {test_run} results into full results") + + test_run_foldername = f"{main_results_folder}{os.sep}run-{test_run}" + _merge_test_reports(test_run_foldername) + + for file in glob.glob(rf"{test_run_foldername}{os.sep}*screenshot*"): + _copy_to_destination_folder(test_run_foldername, file.split(os.sep)[-1], main_results_folder) + + for file in glob.glob(rf"{test_run_foldername}{os.sep}*capture*"): + _copy_to_destination_folder(test_run_foldername, file.split(os.sep)[-1], main_results_folder) + + +def log_report_results(number_of_test_runs: int, failing_suites: []): + logger.info(f"Log available at: file://{os.getcwd()}{os.sep}{main_results_folder}{os.sep}log.html") + logger.info(f"Report available at: file://{os.getcwd()}{os.sep}{main_results_folder}{os.sep}report.html") + logger.info(f"Number of test runs: {number_of_test_runs}") + + if failing_suites: + logger.info(f"Number of failing suites: {len(failing_suites)}") + logger.info(f"Failing suites:") + [logger.info(r" * file://" + suite) for suite in failing_suites] + else: + logger.info("\nAll tests passed!") + + +def create_report_from_output_xml(test_results_folder: str): + create_args = [ + "--outputdir", + f"{test_results_folder}/", + "-o", + "output.xml", + "--xunit", + "xunit.xml", + f"{test_results_folder}/output.xml" + ] + robot_rebot_cli(create_args, exit=False) + + +def _merge_test_reports(test_results_folder): + merge_args = [ + "--outputdir", + f"{main_results_folder}/", + "-o", + "output.xml", + "--xunit", + "xunit.xml", + "--prerebotmodifier", + "report-modifiers/CheckForAtLeastOnePassingRunPrerebotModifier.py", + "--merge", + f"{main_results_folder}/output.xml", + f"{test_results_folder}/output.xml", + ] + robot_rebot_cli(merge_args, exit=False) + + +def filter_out_passing_suites_from_report_file(path_to_original_report: str, path_to_filtered_report: str): + + with open(path_to_original_report, "rb") as report: + + report_contents = report.read() + report = BeautifulSoup(report_contents, features="xml") + + passing_suite_ids = _get_passing_suite_ids_from_report(report) + + suite_elements = report.find_all('suite', recursive=True) + [suite_element.extract() for suite_element in suite_elements if suite_element.get('id') in passing_suite_ids] + + suite_stats = report.find_all('stat', recursive=True) + [suite_stat.extract() for suite_stat in suite_stats if suite_stat.get('id') in passing_suite_ids] + + with open(path_to_filtered_report, "a") as filtered_file: + filtered_file.write(report.prettify()) + + +def _get_passing_suite_ids_from_report(report: BeautifulSoup) -> []: + suite_results = report.find('statistics').find('suite').find_all('stat') + passing_suite_results = [suite_result for suite_result in suite_results if int(suite_result.get('fail')) == 0] + return [passing_suite.get('id') for passing_suite in passing_suite_results] + + +def _copy_to_destination_folder(source_folder: str, source_file: str, destination_folder: str): + source_file_path=f"{source_folder}{os.sep}{source_file}" + destination_file_path=f"{destination_folder}{os.sep}{source_file}" + if os.path.isfile(source_file_path): + shutil.copy(source_file_path, destination_file_path) + else: + shutil.copytree(source_file_path, destination_file_path, dirs_exist_ok=True) diff --git a/tests/robot-tests/run_tests.py b/tests/robot-tests/run_tests.py index a304a3c04f0..9336ea37321 100755 --- a/tests/robot-tests/run_tests.py +++ b/tests/robot-tests/run_tests.py @@ -6,8 +6,8 @@ Run 'python run_tests.py -h' to see argument options """ -import argparse import datetime +import glob import os import random import shutil @@ -19,17 +19,16 @@ import admin_api as admin_api import args_and_variables as args_and_variables import tests.libs.selenium_elements as selenium_elements -from pabot.pabot import main_program as pabot_run_cli -from robot import rebot_cli as robot_rebot_cli -from robot import run_cli as robot_run_cli from scripts.get_webdriver import get_webdriver from tests.libs.create_emulator_release_files import ReleaseFilesGenerator from tests.libs.fail_fast import failing_suites_filename, get_failing_test_suites from tests.libs.logger import get_logger from tests.libs.slack import SlackService +import reports +import test_runners pabot_suite_names_filename = ".pabotsuitenames" -results_foldername = "test-results" +main_results_folder = "test-results" seed_data_files_filepath = "tests/files/seed-data-files.zip" unzipped_seed_data_folderpath = "tests/files/.unzipped-seed-data-files" @@ -61,103 +60,17 @@ def install_chromedriver(chromedriver_version: str): get_webdriver(chromedriver_version) -def create_robot_arguments(arguments: argparse.Namespace, rerunning_failed: bool) -> []: - robot_args = [ - "--outputdir", - f"{results_foldername}/", - "--exclude", - "Failing", - "--exclude", - "UnderConstruction", - "--exclude", - "VisualTesting", - "--xunit", - "xunit", - ] - robot_args += ["-v", f"timeout:{os.getenv('TIMEOUT')}", "-v", f"implicit_wait:{os.getenv('IMPLICIT_WAIT')}"] - if arguments.fail_fast: - robot_args += ["--exitonfailure"] - if arguments.tags: - robot_args += ["--include", arguments.tags] - if arguments.print_keywords: - robot_args += ["--listener", "listeners/KeywordListener.py"] - if arguments.ci: - # NOTE(mark): Ensure secrets aren't visible in CI logs/reports - robot_args += ["--removekeywords", "name:operatingsystem.environment variable should be set"] - robot_args += ["--removekeywords", "name:common.user goes to url"] # To hide basic auth credentials - if arguments.reseed: - robot_args += ["--include", "SeedDataGeneration"] - else: - robot_args += ["--exclude", "SeedDataGeneration"] - if arguments.env == "local": - robot_args += ["--include", "Local", "--exclude", "NotAgainstLocal"] - # seed Azure storage emulator release files - generator = ReleaseFilesGenerator() - generator.create_public_release_files() - generator.create_private_release_files() - if arguments.env == "dev": - robot_args += ["--include", "Dev", "--exclude", "NotAgainstDev"] - if arguments.env == "test": - robot_args += ["--include", "Test", "--exclude", "NotAgainstTest", "--exclude", "AltersData"] - # fmt off - if arguments.env == "preprod": - robot_args += ["--include", "Preprod", "--exclude", "AltersData", "--exclude", "NotAgainstPreProd"] - # fmt on - if arguments.env == "prod": - robot_args += ["--include", "Prod", "--exclude", "AltersData", "--exclude", "NotAgainstProd"] - if arguments.visual: - robot_args += ["-v", "headless:0"] - else: - robot_args += ["-v", "headless:1"] - if os.getenv("RELEASE_COMPLETE_WAIT"): - robot_args += ["-v", f"release_complete_wait:{os.getenv('RELEASE_COMPLETE_WAIT')}"] - if os.getenv("FAIL_TEST_SUITES_FAST"): - robot_args += ["-v", f"FAIL_TEST_SUITES_FAST:{os.getenv('FAIL_TEST_SUITES_FAST')}"] - if arguments.prompt_to_continue: - robot_args += ["-v", "prompt_to_continue_on_failure:1"] - if arguments.debug: - robot_args += ["--loglevel", "DEBUG"] - robot_args += ["-v", "browser:" + arguments.browser] - # We want to add arguments on the first rerun attempt, but on subsequent attempts, we just want - # to change rerunfailedsuites xml file we use - if rerunning_failed: - robot_args += ["--rerunfailedsuites", f"{results_foldername}/output.xml", "--output", "rerun.xml"] - else: - robot_args += ["--output", "output.xml"] - - robot_args += [arguments.tests] - - return robot_args - - def create_run_identifier(): # Add randomness to prevent multiple simultaneous run_tests.py generating the same run_identifier value random_str = "".join([random.choice(string.ascii_lowercase + string.digits) for n in range(6)]) return datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S-" + random_str) -def merge_test_reports(): - merge_args = [ - "--outputdir", - f"{results_foldername}/", - "-o", - "output.xml", - "--xunit", - "xunit.xml", - "--prerebotmodifier", - "report-modifiers/CheckForAtLeastOnePassingRunPrerebotModifier.py", - "--merge", - f"{results_foldername}/output.xml", - f"{results_foldername}/rerun.xml", - ] - robot_rebot_cli(merge_args, exit=False) - - def clear_files_before_test_run(rerunning_failures: bool): # Remove any existing test results if running from scratch. Leave in place if re-running failures # as we'll need the old results to merge in with the rerun results. - if not rerunning_failures and Path(results_foldername).exists(): - shutil.rmtree(results_foldername) + if not rerunning_failures and Path(main_results_folder).exists(): + shutil.rmtree(main_results_folder) # Remove any prior failing suites so the new test run is not marking any running test suites as # failed already. @@ -170,18 +83,20 @@ def clear_files_before_test_run(rerunning_failures: bool): os.remove(pabot_suite_names_filename) -def execute_tests(arguments: argparse.Namespace, rerunning_failures: bool): - if arguments.interp == "robot": - robot_run_cli(create_robot_arguments(arguments, rerunning_failures), exit=False) - elif arguments.interp == "pabot": - pabot_run_cli(create_robot_arguments(arguments, rerunning_failures)) +def run(): + robot_tests_folder = Path(__file__).absolute().parent + + args = args_and_variables.initialise() -def run(): # Unzip any data files that may be used in tests. unzip_data_files() - args = args_and_variables.initialise() + # Upload test data files to storage if running locally. + if args.env == "local": + files_generator = ReleaseFilesGenerator() + files_generator.create_public_release_files() + files_generator.create_private_release_files() install_chromedriver(args.chromedriver_version) @@ -190,24 +105,33 @@ def run(): logger.info(f"Running Robot tests with {args.rerun_attempts} rerun attempts for any failing suites") + if args.rerun_failed_suites: + logger.info(f"Clearing old run folders prior to rerunning failed tests") + for old_run_folders in glob.glob(rf"{main_results_folder}{os.sep}run-*"): + shutil.rmtree(old_run_folders) + try: # Run tests while args.rerun_attempts is None or test_run_index < args.rerun_attempts: try: test_run_index += 1 - # Ensure all SeleniumLibrary elements and keywords are updated to use a branch new + # Ensure all SeleniumLibrary elements and keywords are updated to use a brand new # Selenium instance for every test (re)run. if test_run_index > 0: selenium_elements.clear_instances() rerunning_failed_suites = args.rerun_failed_suites or test_run_index > 0 - + # Perform any cleanup before the test run. clear_files_before_test_run(rerunning_failed_suites) - if not Path(f"{results_foldername}/downloads").exists(): - os.makedirs(f"{results_foldername}/downloads") + # Create a folder to contain this test run attempt's outputs and reports. + test_run_results_folder = f"{main_results_folder}{os.sep}run-{test_run_index + 1}" + os.makedirs(test_run_results_folder) + + if not Path(f"{main_results_folder}/downloads").exists(): + os.makedirs(f"{main_results_folder}/downloads") # Create a unique run identifier so that this test run's data will be unique. run_identifier = f"{run_identifier_initial_value}-{test_run_index}" @@ -217,16 +141,26 @@ def run(): if args_and_variables.includes_data_changing_tests(args): admin_api.create_test_topic(run_identifier) - # Run the tests. - logger.info(f"Performing test run {test_run_index + 1} with unique identifier {run_identifier}") - execute_tests(args, rerunning_failed_suites) - - # If we're rerunning failures, merge the former run's results with this run's - # results. + # If re-running failed suites, get the appropriate previous report file from which to determine which suites failed. + # If this is run 2 or more when using the `--rerun-attempts n` option, the path to the previous report will be in the + # previous run attempt's folder. + # If this is a rerun using the `--rerun-failed-suites` option, the path to the previous report will be in the main + # test results folder directly. if rerunning_failed_suites: - logger.info(f"Merging results from test run {test_run_index + 1} with previous run's report") - merge_test_reports() + if test_run_index == 0: + previous_report_file = f"{robot_tests_folder}{os.sep}{main_results_folder}{os.sep}output.xml" + else: + previous_report_file = f"{robot_tests_folder}{os.sep}{main_results_folder}{os.sep}run-{test_run_index}{os.sep}output.xml" + else: + previous_report_file = None + # Run the tests. + logger.info(f"Performing test run {test_run_index + 1} with unique identifier {run_identifier}") + test_runners.execute_tests(args, test_run_results_folder, previous_report_file) + + if test_run_index > 0: + reports.create_report_from_output_xml(test_run_results_folder) + finally: # Tear down any data created by this test run unless we've disabled teardown. if args_and_variables.includes_data_changing_tests(args) and not args.disable_teardown: @@ -237,30 +171,25 @@ def run(): if not get_failing_test_suites(): break - logger.info(f"Log available at: file://{os.getcwd()}{os.sep}{results_foldername}{os.sep}log.html") - logger.info(f"Report available at: file://{os.getcwd()}{os.sep}{results_foldername}{os.sep}report.html") - - logger.info(f"Number of test runs: {test_run_index + 1}") - failing_suites = get_failing_test_suites() - if failing_suites: - logger.info(f"Number of failing suites: {len(failing_suites)}") - logger.info(f"Failing suites:") - [logger.info(r" * file://" + suite) for suite in failing_suites] - else: - logger.info("\nAll tests passed!") + # Merge together all reports from all test runs. + number_of_test_runs = test_run_index + 1 + reports.merge_robot_reports(number_of_test_runs) + + # Log the results of the merge test runs. + reports.log_report_results(number_of_test_runs, failing_suites) if args.enable_slack_notifications: slack_service = SlackService() # Wait for 5 seconds to ensure the merge reports are properly synchronized after rerun attempts. time.sleep(5) - slack_service.send_test_report(args.env, args.tests, failing_suites, test_run_index) + slack_service.send_test_report(args.env, args.tests, failing_suites, number_of_test_runs) except Exception as ex: if args.enable_slack_notifications: slack_service = SlackService() - slack_service.send_exception_details(args.env, args.tests, test_run_index, ex) + slack_service.send_exception_details(args.env, args.tests, number_of_test_runs, ex) raise ex diff --git a/tests/robot-tests/test_runners.py b/tests/robot-tests/test_runners.py new file mode 100644 index 00000000000..4567612050c --- /dev/null +++ b/tests/robot-tests/test_runners.py @@ -0,0 +1,110 @@ +import os +import reports +import argparse +import pabot.pabot as pabot +import robot +from tests.libs.logger import get_logger + +logger = get_logger(__name__) + + +def create_robot_arguments(arguments: argparse.Namespace, test_run_folder: str) -> []: + robot_args = [ + "--name", + "UI Tests", + "--outputdir", + f"{test_run_folder}/", + "--exclude", + "Failing", + "--exclude", + "UnderConstruction", + "--exclude", + "VisualTesting", + "--xunit", + "xunit", + ] + + robot_args += ["-v", f"timeout:{os.getenv('TIMEOUT')}", "-v", f"implicit_wait:{os.getenv('IMPLICIT_WAIT')}"] + + if arguments.fail_fast: + robot_args += ["--exitonfailure"] + if arguments.tags: + robot_args += ["--include", arguments.tags] + if arguments.print_keywords: + robot_args += ["--listener", "listeners/KeywordListener.py"] + if arguments.ci: + # NOTE(mark): Ensure secrets aren't visible in CI logs/reports + robot_args += ["--removekeywords", "name:operatingsystem.environment variable should be set"] + robot_args += ["--removekeywords", "name:common.user goes to url"] # To hide basic auth credentials + + process_includes_and_excludes(robot_args, arguments) + + if arguments.visual: + robot_args += ["-v", "headless:0"] + else: + robot_args += ["-v", "headless:1"] + if os.getenv("RELEASE_COMPLETE_WAIT"): + robot_args += ["-v", f"release_complete_wait:{os.getenv('RELEASE_COMPLETE_WAIT')}"] + if arguments.prompt_to_continue: + robot_args += ["-v", "prompt_to_continue_on_failure:1"] + if arguments.debug: + robot_args += ["--loglevel", "DEBUG"] + robot_args += ["-v", "browser:" + arguments.browser] + # We want to add arguments on the first rerun attempt, but on subsequent attempts, we just want + # to change rerunfailedsuites xml file we use + robot_args += ["--output", "output.xml"] + + return robot_args + + +def process_includes_and_excludes(robot_args: [], arguments: argparse.Namespace): + if arguments.reseed: + robot_args += ["--include", "SeedDataGeneration"] + else: + robot_args += ["--exclude", "SeedDataGeneration"] + if arguments.env == "local": + robot_args += ["--include", "Local", "--exclude", "NotAgainstLocal"] + if arguments.env == "dev": + robot_args += ["--include", "Dev", "--exclude", "NotAgainstDev"] + if arguments.env == "test": + robot_args += ["--include", "Test", "--exclude", "NotAgainstTest", "--exclude", "AltersData"] + # fmt off + if arguments.env == "preprod": + robot_args += ["--include", "Preprod", "--exclude", "AltersData", "--exclude", "NotAgainstPreProd"] + # fmt on + if arguments.env == "prod": + robot_args += ["--include", "Prod", "--exclude", "AltersData", "--exclude", "NotAgainstProd"] + + +def execute_tests(arguments: argparse.Namespace, test_run_folder: str, path_to_previous_report_file: str): + + robot_args = create_robot_arguments(arguments, test_run_folder) + + if arguments.interp == "robot": + + if path_to_previous_report_file is not None: + robot_args += ["--rerunfailedsuites", path_to_previous_report_file] + + robot_args += [arguments.tests] + + robot.run_cli(robot_args, exit=False) + + elif arguments.interp == "pabot": + + logger.info('Performing test run with Pabot') + + if path_to_previous_report_file is not None: + + logger.info(f'Re-running failed suites from {path_to_previous_report_file}') + + path_to_filtered_report_file = '_filtered.xml'.join(path_to_previous_report_file.rsplit('.xml', 1)) + + reports.filter_out_passing_suites_from_report_file(path_to_previous_report_file, path_to_filtered_report_file) + + logger.info(f'Generated filtered report file containing only failing suites at {path_to_filtered_report_file}') + + robot_args = ['--suitesfrom', path_to_filtered_report_file] + robot_args + + robot_args += [arguments.tests] + + pabot.main_program(robot_args) diff --git a/tests/robot-tests/tests/general_public/data_set_page_absence_in_prus.robot b/tests/robot-tests/tests/general_public/data_set_page_absence_in_prus.robot index 00f526f6667..1766d414dc9 100644 --- a/tests/robot-tests/tests/general_public/data_set_page_absence_in_prus.robot +++ b/tests/robot-tests/tests/general_public/data_set_page_absence_in_prus.robot @@ -88,4 +88,4 @@ Validate table tool link user checks page contains Choose locations user checks previous table tool step contains 1 Publication ${PUPIL_ABSENCE_PUBLICATION_TITLE} - user checks previous table tool step contains 2 Data set Absence in PRUs \ No newline at end of file + user checks previous table tool step contains 2 Data set Absence in PRUs diff --git a/tests/robot-tests/tests/libs/admin/manage-content-common.robot b/tests/robot-tests/tests/libs/admin/manage-content-common.robot index e6dcb2aba73..92c9ae400b3 100644 --- a/tests/robot-tests/tests/libs/admin/manage-content-common.robot +++ b/tests/robot-tests/tests/libs/admin/manage-content-common.robot @@ -468,12 +468,12 @@ user adds image to accordion section text block with retry ... ${FILES_DIR}${filename} user scrolls up 300 - wait until keyword succeeds ${timeout} 1 sec user clicks button Change image text alternative + wait until keyword succeeds ${timeout} %{WAIT_SMALL} sec user clicks button Change image text alternative user enters text into element label:Text alternative ${alt_text} user clicks element css:button.ck-button-save sleep 5 user scrolls up 100 - wait until keyword succeeds ${timeout} 1 sec user clicks element xpath://div[@title="Insert paragraph after block"] + wait until keyword succeeds ${timeout} %{WAIT_SMALL} sec user clicks element xpath://div[@title="Insert paragraph after block"] # wait for the API to save the image and for the src attribute to be updated before continuing user waits until parent contains element ${block} diff --git a/tests/robot-tests/tests/libs/common.robot b/tests/robot-tests/tests/libs/common.robot index 6bb87a3410b..4ce02a7483e 100644 --- a/tests/robot-tests/tests/libs/common.robot +++ b/tests/robot-tests/tests/libs/common.robot @@ -1,5 +1,5 @@ *** Settings *** -Library SeleniumLibrary timeout=%{TIMEOUT} implicit_wait=%{IMPLICIT_WAIT} run_on_failure=do this on failure +Library SeleniumLibrary timeout=%{TIMEOUT} implicit_wait=%{IMPLICIT_WAIT} run_on_failure=record test failure Library OperatingSystem Library Collections Library file_operations.py @@ -20,31 +20,10 @@ ${DOWNLOADS_DIR}= ${EXECDIR}${/}test-results${/}downloads$ ${timeout}= %{TIMEOUT} ${implicit_wait}= %{IMPLICIT_WAIT} ${prompt_to_continue_on_failure}= 0 -${FAIL_TEST_SUITES_FAST}= %{FAIL_TEST_SUITES_FAST} ${DATE_FORMAT_MEDIUM}= %-d %B %Y *** Keywords *** -do this on failure - # See if the currently executing Test Suite is failing fast and if not, take a screenshot and HTML grab of the - # failing page. - ${currently_failing_fast}= current test suite failing fast - - IF "${currently_failing_fast}" == "${FALSE}" - capture screenshots and html - - # Additionally, mark the current Test Suite as failing if the "FAIL_TEST_SUITES_FAST" option is enabled, and - # this will cause subsequent tests within this same Test Suite to fail immediately (by virtue of their "Test - # Setup" steps checking to see if their owning Test Suite is currently failing fast). - IF ${FAIL_TEST_SUITES_FAST} == 1 - record failing test suite - END - END - - IF ${prompt_to_continue_on_failure} == 1 - prompt to continue - END - user opens the browser [Arguments] ${alias}=main IF "${browser}" == "chrome" diff --git a/tests/robot-tests/tests/libs/fail_fast.py b/tests/robot-tests/tests/libs/fail_fast.py index 224d34353b7..c9f075e514f 100644 --- a/tests/robot-tests/tests/libs/fail_fast.py +++ b/tests/robot-tests/tests/libs/fail_fast.py @@ -5,18 +5,31 @@ should continue to run or if they should fail immediately and therefore fail the test suite immediately. """ +import datetime import os.path import os import threading from robot.libraries.BuiltIn import BuiltIn +from robot.api import SkipExecution from tests.libs.logger import get_logger from tests.libs.selenium_elements import sl +import tests.libs.visual as visual failing_suites_filename = ".failing_suites" logger = get_logger(__name__) +def record_test_failure(): + if not current_test_suite_failing_fast(): + record_failing_test_suite() + visual.capture_screenshot() + visual.capture_large_screenshot() + _capture_html() + + if BuiltIn().get_variable_value("${prompt_to_continue_on_failure}") == '1': + _prompt_to_continue() + def _get_current_test_suite() -> str: return BuiltIn().get_variable_value("${SUITE SOURCE}") @@ -32,10 +45,15 @@ def current_test_suite_failing_fast() -> bool: def record_failing_test_suite(): + if current_test_suite_failing_fast(): + return + test_suite = _get_current_test_suite() - logger.warn( + + logger.info( f"Recording test suite '{test_suite}' as failing - subsequent tests will automatically fail in this suite" ) + with file_lock: try: with open(failing_suites_filename, "a") as file_write: @@ -46,32 +64,37 @@ def record_failing_test_suite(): def fail_test_fast_if_required(): if current_test_suite_failing_fast(): - _raise_assertion_error(f"Test suite {_get_current_test_suite()} is already failing. Failing this test fast.") + raise SkipExecution(f"Test suite {_get_current_test_suite()} is already failing. Skipping this test.") def get_failing_test_suites() -> []: - if os.path.isfile(failing_suites_filename): - # We wouldn't expect the same test suite to be recorded in this file more than once, as we only trigger the - # "record failing test suite" upon the first failing test in an individual suite. - # - # Strangely though, this does get called multiple times if using Pabot and re-running failed suites. It seems as - # though the failure keywords are being merged with the initial run's failure keyword definitions and therefore - # causing the failing test suite to be recorded multiple times when its first failing test is hit. - # - # We therefore explicitly remove any duplicates from the list here. - - with file_lock: + with file_lock: + if os.path.isfile(failing_suites_filename): try: with open(failing_suites_filename, "r") as file: - failing_suites = file.readlines() - stripped_suite_names = [failing_suite.strip() for failing_suite in failing_suites] - filtered_suite_names = filter(None, stripped_suite_names) - return list(dict.fromkeys(filtered_suite_names)) + return [suite.strip() for suite in file.readlines()] except IOError as e: logger.error(f"Failed to read failing test suites from file: {e}") return [] - return [] + return [] + + +def _capture_html(): + html = sl().get_source() + current_time_millis = round(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) + output_dir = BuiltIn().get_variable_value('${OUTPUT DIR}') + html_file = open(f"{output_dir}{os.sep}captured-html-{current_time_millis}.html", "w", encoding="utf-8") + html_file.write(html) + html_file.close() + logger.info(f"Captured HTML of {sl().get_location()} HTML saved to file://{os.path.realpath(html_file.name)}") + +def _prompt_to_continue(): + logger.warn("Continue? (Y/n)") + choice = input() + if choice.lower().startswith("n"): + raise_assertion_error("Tests stopped!") + def _raise_assertion_error(err_msg): sl().failure_occurred() diff --git a/tests/robot-tests/tests/libs/slack.py b/tests/robot-tests/tests/libs/slack.py index 1e092dfd170..31536dfc2b6 100644 --- a/tests/robot-tests/tests/libs/slack.py +++ b/tests/robot-tests/tests/libs/slack.py @@ -31,11 +31,12 @@ def _build_test_results_attachments(self, env: str, suites_ran: str, suites_fail tests = soup.find("total").find("stat") failed_tests = int(tests["fail"]) passed_tests = int(tests["pass"]) + skipped_tests = int(tests["skip"]) except AttributeError as e: raise Exception("Error parsing the XML report") from e - total_tests_count = passed_tests + failed_tests + total_tests_count = passed_tests + failed_tests + skipped_tests blocks = [ { @@ -54,6 +55,7 @@ def _build_test_results_attachments(self, env: str, suites_ran: str, suites_fail {"type": "mrkdwn", "text": f"*Total test cases*\n{total_tests_count}"}, {"type": "mrkdwn", "text": f"*Passed test cases*\n{passed_tests}"}, {"type": "mrkdwn", "text": f"*Failed test cases*\n{failed_tests}"}, + {"type": "mrkdwn", "text": f"*Skipped test cases*\n{skipped_tests}"}, ], }, ] @@ -79,7 +81,7 @@ def _build_test_results_attachments(self, env: str, suites_ran: str, suites_fail return blocks - def _build_exception_details_attachments(self, env: str, suites_ran: str, run_index: int, ex: Exception): + def _build_exception_details_attachments(self, env: str, suites_ran: str, number_of_test_runs: int, ex: Exception): ex_stripped = repr(ex).replace("\n", "") suites_ran_conditional = "N/A" if not suites_ran else suites_ran.replace("tests/", "") @@ -90,15 +92,15 @@ def _build_exception_details_attachments(self, env: str, suites_ran: str, run_in "fields": [ {"type": "mrkdwn", "text": f"*Environment*\n{env}"}, {"type": "mrkdwn", "text": f"*Suite*\n{suites_ran_conditional}"}, - {"type": "mrkdwn", "text": f"*Run number*\n{run_index + 1}"}, + {"type": "mrkdwn", "text": f"*Run number*\n{number_of_test_runs}"}, ], }, {"type": "divider"}, {"type": "section", "text": {"type": "mrkdwn", "text": f"*Error details*\n{ex_stripped}"}}, ] - def send_test_report(self, env: str, suites_ran: str, suites_failed: [], run_index: int): - attachments = self._build_test_results_attachments(env, suites_ran, suites_failed, run_index) + def send_test_report(self, env: str, suites_ran: str, suites_failed: [], number_of_test_runs: int): + attachments = self._build_test_results_attachments(env, suites_ran, suites_failed, number_of_test_runs) response = self.client.chat_postMessage(channel=self.slack_channel, text="All results", blocks=attachments) @@ -121,8 +123,8 @@ def send_test_report(self, env: str, suites_ran: str, suites_failed: [], run_ind logger.info("Sent UI test statistics to #build") - def send_exception_details(self, env: str, suites_ran: str, run_index: int, ex: Exception): - attachments = self._build_exception_details_attachments(env, suites_ran, run_index, ex) + def send_exception_details(self, env: str, suites_ran: str, number_of_test_runs: int, ex: Exception): + attachments = self._build_exception_details_attachments(env, suites_ran, number_of_test_runs, ex) response = self.client.chat_postMessage( text=":x: UI test pipeline failure", channel=self.slack_channel, blocks=attachments diff --git a/tests/robot-tests/tests/libs/utilities.py b/tests/robot-tests/tests/libs/utilities.py index ea2782e19eb..ed25043690a 100644 --- a/tests/robot-tests/tests/libs/utilities.py +++ b/tests/robot-tests/tests/libs/utilities.py @@ -63,8 +63,6 @@ def enable_basic_auth_headers(): token = base64.b64encode(f"{public_auth_user}:{public_auth_password}".encode()) try: - # Must refetch sl() on rerun or sl().driver is None! - # sl() = BuiltIn().get_library_instance("SeleniumLibrary") assert sl().driver is not None, "sl().driver is None" sl().driver.execute_cdp_cmd("Network.enable", {}) @@ -92,7 +90,7 @@ def retry_or_fail_with_delay(func, retries=5, delay=1.0, *args, **kwargs): return func(*args, **kwargs) except Exception as e: last_exception = e - logger.warn(f"Attempt {attempt + 1}/{retries} failed with error: {e}. Retrying in {delay} seconds...") + logger.info(f"Attempt {attempt + 1}/{retries} failed with error: {e}. Retrying in {delay} seconds...") time.sleep(delay) # Raise the last exception if all retries failed raise last_exception @@ -108,10 +106,13 @@ def user_waits_until_parent_contains_element( delay: float = 1.0, ): try: + default_timeout = BuiltIn().get_variable_value('${TIMEOUT}') + timeout_per_retry = timeout / retries if timeout is not None else int(default_timeout) / retries + child_locator = _normalise_child_locator(child_locator) def parent_contains_matching_element() -> bool: - parent_el = _get_parent_webelement_from_locator(parent_locator, timeout, error) + parent_el = _get_parent_webelement_from_locator(parent_locator, timeout_per_retry, error) return element_finder().find(child_locator, required=False, parent=parent_el) is not None if is_noney(count): @@ -121,14 +122,14 @@ def parent_contains_matching_element() -> bool: delay, parent_contains_matching_element, "Parent '%s' did not contain '%s' in ." % (parent_locator, child_locator), - timeout, + timeout_per_retry, error, ) count = int(count) def parent_contains_matching_elements() -> bool: - parent_el = _get_parent_webelement_from_locator(parent_locator, timeout, error) + parent_el = _get_parent_webelement_from_locator(parent_locator, timeout_per_retry, error) return len(sl().find_elements(child_locator, parent=parent_el)) == count retry_or_fail_with_delay( @@ -137,7 +138,7 @@ def parent_contains_matching_elements() -> bool: delay, parent_contains_matching_elements, "Parent '%s' did not contain %s '%s' element(s) within ." % (parent_locator, count, child_locator), - timeout, + timeout_per_retry, error, ) except Exception as err: @@ -304,33 +305,11 @@ def user_should_be_at_top_of_page(): raise_assertion_error(f"Windows position Y is {y} not 0! User should be at the top of the page!") -def prompt_to_continue(): - logger.warn("Continue? (Y/n)") - choice = input() - if choice.lower().startswith("n"): - raise_assertion_error("Tests stopped!") - - def capture_large_screenshot_and_prompt_to_continue(): visual.capture_large_screenshot() prompt_to_continue() -def capture_screenshots_and_html(): - visual.capture_screenshot() - visual.capture_large_screenshot() - capture_html() - - -def capture_html(): - html = sl().get_source() - current_time_millis = round(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) - html_file = open(f"test-results/captured-html-{current_time_millis}.html", "w", encoding="utf-8") - html_file.write(html) - html_file.close() - logger.warn(f"Captured HTML of {sl().get_location()} HTML saved to file://{os.path.realpath(html_file.name)}") - - def user_gets_row_number_with_heading(heading: str, table_locator: str = "css:table"): elem = get_child_element(table_locator, f'xpath:.//tbody/tr/th[text()="{heading}"]/..') rows = get_child_elements(table_locator, "css:tbody tr") diff --git a/tests/robot-tests/tests/libs/visual.py b/tests/robot-tests/tests/libs/visual.py index ba3417669a3..b7e237a3ee5 100644 --- a/tests/robot-tests/tests/libs/visual.py +++ b/tests/robot-tests/tests/libs/visual.py @@ -46,7 +46,7 @@ def highlight_element(element: WebElement): def capture_screenshot(): screenshot_location = sl().capture_page_screenshot() - logger.warn( + logger.info( f"Captured current screenshot at URL '{sl().get_location()}' Screenshot saved to file://{screenshot_location}" ) @@ -54,7 +54,7 @@ def capture_screenshot(): @with_maximised_browser def capture_large_screenshot(): screenshot_location = sl().capture_page_screenshot() - logger.warn( + logger.info( f"Captured enlarged screenshot at URL '{sl().get_location()}' Screenshot saved to file://{screenshot_location}" ) diff --git a/tests/robot-tests/tests/public_api/public_api_cancel_and_removal.robot b/tests/robot-tests/tests/public_api/public_api_cancel_and_removal.robot index 5675746d6fe..9ccec2f27c8 100644 --- a/tests/robot-tests/tests/public_api/public_api_cancel_and_removal.robot +++ b/tests/robot-tests/tests/public_api/public_api_cancel_and_removal.robot @@ -79,7 +79,7 @@ User creates 2nd API dataset User waits until the 2nd API dataset status changes to 'Ready' user waits until h3 is visible Draft version details - wait until keyword succeeds 10x 5s Verify status of API Datasets Ready + wait until keyword succeeds 10x %{WAIT_SMALL}s Verify status of API Datasets Ready Verify the contents inside the 'Draft API datasets' table user clicks link Back to API data sets @@ -135,7 +135,7 @@ User creates 1st API dataset again User waits until the 1st API dataset status changes to 'Ready' user waits until h3 is visible Draft version details - wait until keyword succeeds 10x 5s Verify status of API Datasets Ready + wait until keyword succeeds 10x %{WAIT_SMALL}s Verify status of API Datasets Ready Add headline text block to Content page user navigates to content page ${PUBLICATION_NAME} diff --git a/tests/robot-tests/tests/public_api/public_api_preview_token.robot b/tests/robot-tests/tests/public_api/public_api_preview_token.robot index 325aae7bad6..71f74e9ea22 100644 --- a/tests/robot-tests/tests/public_api/public_api_preview_token.robot +++ b/tests/robot-tests/tests/public_api/public_api_preview_token.robot @@ -70,7 +70,7 @@ Create 1st API dataset User waits until the 1st API dataset status changes to 'Ready' user waits until h3 is visible Draft version details - wait until keyword succeeds 10x 5s Verify status of API Datasets Ready + wait until keyword succeeds 10x %{WAIT_SMALL}s Verify status of API Datasets Ready Create 2nd API dataset user clicks link Back to API data sets @@ -84,7 +84,7 @@ Create 2nd API dataset User waits until the 2nd API dataset status changes to 'Ready' user waits until h3 is visible Draft version details - wait until keyword succeeds 10x 5s Verify status of API Datasets Ready + wait until keyword succeeds 10x %{WAIT_SMALL}s Verify status of API Datasets Ready Verify the contents inside the 'Draft API datasets' table user clicks link Back to API data sets diff --git a/tests/robot-tests/tests/public_api/public_api_resolve_mapping_statuses.robot b/tests/robot-tests/tests/public_api/public_api_resolve_mapping_statuses.robot index cc8f3b336c7..a8b6ac1cb04 100644 --- a/tests/robot-tests/tests/public_api/public_api_resolve_mapping_statuses.robot +++ b/tests/robot-tests/tests/public_api/public_api_resolve_mapping_statuses.robot @@ -66,7 +66,7 @@ Create 1st API dataset User waits until the 1st API dataset status changes to 'Ready' user waits until h3 is visible Draft version details - wait until keyword succeeds 10x 5s Verify status of API Datasets Ready + wait until keyword succeeds 10x %{WAIT_SMALL}s Verify status of API Datasets Ready Add headline text block to Content page user clicks link Back to API data sets diff --git a/tests/robot-tests/tests/public_api/public_api_restricted.robot b/tests/robot-tests/tests/public_api/public_api_restricted.robot index 76375cf9a5b..7d50cdeb648 100644 --- a/tests/robot-tests/tests/public_api/public_api_restricted.robot +++ b/tests/robot-tests/tests/public_api/public_api_restricted.robot @@ -73,7 +73,7 @@ Create 1st API dataset User waits until the 1st API dataset status changes to 'Ready' user waits until h3 is visible Draft version details - wait until keyword succeeds 10x 5s Verify status of API Datasets Ready + wait until keyword succeeds 10x %{WAIT_SMALL}s Verify status of API Datasets Ready Create 2nd API dataset user clicks link Back to API data sets @@ -87,7 +87,7 @@ Create 2nd API dataset User waits until the 2nd API dataset status changes to 'Ready' user waits until h3 is visible Draft version details - wait until keyword succeeds 10x 5s Verify status of API Datasets Ready + wait until keyword succeeds 10x %{WAIT_SMALL}s Verify status of API Datasets Ready Verify the contents inside the 'Draft API datasets' table user clicks link Back to API data sets @@ -183,7 +183,7 @@ Create a new API dataset version through the first amendment using the invalid s User waits until the 2nd invalid API dataset status changes to 'Failed' user waits until h3 is visible Draft version details - wait until keyword succeeds 20x 5s Verify status of API Datasets Failed + wait until keyword succeeds 20x %{WAIT_SMALL}s Verify status of API Datasets Failed Verify the contents inside the 'Draft API datasets' table after the invalid import fails user clicks link Back to API data sets From 64af798609a69d35aa88053ad4e85c9cb50a396a Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 2 Oct 2024 11:51:33 +0100 Subject: [PATCH 32/80] EES-5540 - responding to PR comments from the one and only Luke Howsam! Adding additional details to Pabot logging. Improving arg processing code. --- tests/robot-tests/test_runners.py | 38 ++++++++++++++----------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/tests/robot-tests/test_runners.py b/tests/robot-tests/test_runners.py index 4567612050c..5f92496142c 100644 --- a/tests/robot-tests/test_runners.py +++ b/tests/robot-tests/test_runners.py @@ -14,22 +14,16 @@ def create_robot_arguments(arguments: argparse.Namespace, test_run_folder: str) "UI Tests", "--outputdir", f"{test_run_folder}/", - "--exclude", - "Failing", - "--exclude", - "UnderConstruction", - "--exclude", - "VisualTesting", "--xunit", "xunit", ] + robot_args += _create_include_and_exclude_args(arguments) + robot_args += ["-v", f"timeout:{os.getenv('TIMEOUT')}", "-v", f"implicit_wait:{os.getenv('IMPLICIT_WAIT')}"] if arguments.fail_fast: robot_args += ["--exitonfailure"] - if arguments.tags: - robot_args += ["--include", arguments.tags] if arguments.print_keywords: robot_args += ["--listener", "listeners/KeywordListener.py"] if arguments.ci: @@ -37,14 +31,10 @@ def create_robot_arguments(arguments: argparse.Namespace, test_run_folder: str) robot_args += ["--removekeywords", "name:operatingsystem.environment variable should be set"] robot_args += ["--removekeywords", "name:common.user goes to url"] # To hide basic auth credentials - process_includes_and_excludes(robot_args, arguments) - if arguments.visual: robot_args += ["-v", "headless:0"] else: robot_args += ["-v", "headless:1"] - if os.getenv("RELEASE_COMPLETE_WAIT"): - robot_args += ["-v", f"release_complete_wait:{os.getenv('RELEASE_COMPLETE_WAIT')}"] if arguments.prompt_to_continue: robot_args += ["-v", "prompt_to_continue_on_failure:1"] if arguments.debug: @@ -57,23 +47,29 @@ def create_robot_arguments(arguments: argparse.Namespace, test_run_folder: str) return robot_args -def process_includes_and_excludes(robot_args: [], arguments: argparse.Namespace): +def _create_include_and_exclude_args(arguments: argparse.Namespace) -> []: + include_exclude_args = ["--exclude", "Failing"] + include_exclude_args += ["--exclude", "UnderConstruction"] + include_exclude_args += ["--exclude", "VisualTesting"] + if arguments.tags: + include_exclude_args += ["--include", arguments.tags] if arguments.reseed: - robot_args += ["--include", "SeedDataGeneration"] + include_exclude_args += ["--include", "SeedDataGeneration"] else: - robot_args += ["--exclude", "SeedDataGeneration"] + include_exclude_args += ["--exclude", "SeedDataGeneration"] if arguments.env == "local": - robot_args += ["--include", "Local", "--exclude", "NotAgainstLocal"] + include_exclude_args += ["--include", "Local", "--exclude", "NotAgainstLocal"] if arguments.env == "dev": - robot_args += ["--include", "Dev", "--exclude", "NotAgainstDev"] + include_exclude_args += ["--include", "Dev", "--exclude", "NotAgainstDev"] if arguments.env == "test": - robot_args += ["--include", "Test", "--exclude", "NotAgainstTest", "--exclude", "AltersData"] + include_exclude_args += ["--include", "Test", "--exclude", "NotAgainstTest", "--exclude", "AltersData"] # fmt off if arguments.env == "preprod": - robot_args += ["--include", "Preprod", "--exclude", "AltersData", "--exclude", "NotAgainstPreProd"] + include_exclude_args += ["--include", "Preprod", "--exclude", "AltersData", "--exclude", "NotAgainstPreProd"] # fmt on if arguments.env == "prod": - robot_args += ["--include", "Prod", "--exclude", "AltersData", "--exclude", "NotAgainstProd"] + include_exclude_args += ["--include", "Prod", "--exclude", "AltersData", "--exclude", "NotAgainstProd"] + return include_exclude_args def execute_tests(arguments: argparse.Namespace, test_run_folder: str, path_to_previous_report_file: str): @@ -91,7 +87,7 @@ def execute_tests(arguments: argparse.Namespace, test_run_folder: str, path_to_p elif arguments.interp == "pabot": - logger.info('Performing test run with Pabot') + logger.info(f'Performing test run with Pabot ({arguments.processes} processes)') if path_to_previous_report_file is not None: From be57edfd274d0a6140e0aebecaecb35335b1992c Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Thu, 3 Oct 2024 15:07:50 +0100 Subject: [PATCH 33/80] EES-5540 - additional changes to get combinations of the --rerun-failed-suites and --rerun-attempts working together better --- tests/robot-tests/args_and_variables.py | 13 +++- tests/robot-tests/reports.py | 16 +++-- tests/robot-tests/run_tests.py | 90 ++++++++++++++----------- tests/robot-tests/test_runners.py | 18 ++--- 4 files changed, 80 insertions(+), 57 deletions(-) diff --git a/tests/robot-tests/args_and_variables.py b/tests/robot-tests/args_and_variables.py index 994d8bec279..2723e9a93e8 100644 --- a/tests/robot-tests/args_and_variables.py +++ b/tests/robot-tests/args_and_variables.py @@ -27,7 +27,11 @@ def create_argument_parser() -> argparse.ArgumentParser: help="interpreter to use to run the tests", ) parser.add_argument( - "--processes", dest="processes", help="how many processes should be used when using the pabot interpreter" + "--processes", + dest="processes", + type=int, + default=4, + help="how many processes should be used when using the pabot interpreter" ) parser.add_argument( "-e", @@ -78,7 +82,12 @@ def create_argument_parser() -> argparse.ArgumentParser: action="store_true", help="rerun failed test suites and merge results into original run results", ) - parser.add_argument("--rerun-attempts", dest="rerun_attempts", type=int, default=0, help="Number of rerun attempts") + parser.add_argument( + "--rerun-attempts", + dest="rerun_attempts", + type=int, + default=0, + help="Number of rerun attempts") parser.add_argument( "--print-keywords", dest="print_keywords", diff --git a/tests/robot-tests/reports.py b/tests/robot-tests/reports.py index 10f89095f69..56b832e0c8a 100644 --- a/tests/robot-tests/reports.py +++ b/tests/robot-tests/reports.py @@ -1,6 +1,7 @@ import os import glob import shutil +import time from bs4 import BeautifulSoup from robot import rebot_cli as robot_rebot_cli from tests.libs.logger import get_logger @@ -11,14 +12,16 @@ # Merge multiple Robot test reports and assets together into the main test results folder. -def merge_robot_reports(number_of_test_runs: int): +def merge_robot_reports(first_run_attempt_number: int, number_of_test_runs: int): - run_1_folder=f"{main_results_folder}{os.sep}run-1" + first_run_folder=f"{main_results_folder}{os.sep}run-{first_run_attempt_number}" - for file in os.listdir(run_1_folder): - _copy_to_destination_folder(run_1_folder, file, main_results_folder) + logger.info(f"Merging test run {first_run_attempt_number} results into full results") + + for file in os.listdir(first_run_folder): + _copy_to_destination_folder(first_run_folder, file, main_results_folder) - for test_run in range(2, number_of_test_runs + 1): + for test_run in range(first_run_attempt_number + 1, number_of_test_runs + 1): logger.info(f"Merging test run {test_run} results into full results") @@ -90,6 +93,9 @@ def filter_out_passing_suites_from_report_file(path_to_original_report: str, pat suite_stats = report.find_all('stat', recursive=True) [suite_stat.extract() for suite_stat in suite_stats if suite_stat.get('id') in passing_suite_ids] + if os.path.exists(path_to_filtered_report): + os.remove(path_to_filtered_report) + with open(path_to_filtered_report, "a") as filtered_file: filtered_file.write(report.prettify()) diff --git a/tests/robot-tests/run_tests.py b/tests/robot-tests/run_tests.py index 9336ea37321..36b82e76cbc 100755 --- a/tests/robot-tests/run_tests.py +++ b/tests/robot-tests/run_tests.py @@ -6,12 +6,14 @@ Run 'python run_tests.py -h' to see argument options """ +import argparse import datetime import glob import os import random import shutil import string +import sys import time from pathlib import Path from zipfile import ZipFile @@ -35,7 +37,7 @@ logger = get_logger(__name__) -def setup_python_path(): +def _setup_python_path(): # This is super awkward but we have to explicitly # add the current directory to PYTHONPATH otherwise # the subprocesses started by pabot will not be able @@ -47,7 +49,7 @@ def setup_python_path(): os.environ["PYTHONPATH"] = str(current_dir) -def unzip_data_files(): +def _unzip_data_files(): if not os.path.exists(seed_data_files_filepath): logger.warn(f"Unable to find seed data files bundle at {seed_data_files_filepath}") else: @@ -55,23 +57,18 @@ def unzip_data_files(): zipfile.extractall(unzipped_seed_data_folderpath) -def install_chromedriver(chromedriver_version: str): +def _install_chromedriver(chromedriver_version: str): # Install chromedriver and add it to PATH get_webdriver(chromedriver_version) -def create_run_identifier(): +def _create_run_identifier(): # Add randomness to prevent multiple simultaneous run_tests.py generating the same run_identifier value random_str = "".join([random.choice(string.ascii_lowercase + string.digits) for n in range(6)]) return datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S-" + random_str) -def clear_files_before_test_run(rerunning_failures: bool): - # Remove any existing test results if running from scratch. Leave in place if re-running failures - # as we'll need the old results to merge in with the rerun results. - if not rerunning_failures and Path(main_results_folder).exists(): - shutil.rmtree(main_results_folder) - +def _clear_files_before_next_test_run_attempt(rerunning_failures: bool): # Remove any prior failing suites so the new test run is not marking any running test suites as # failed already. if Path(failing_suites_filename).exists(): @@ -83,14 +80,36 @@ def clear_files_before_test_run(rerunning_failures: bool): os.remove(pabot_suite_names_filename) +def _setup_main_results_folder_for_first_run(args: argparse.Namespace): + # If rerunning failing tests from a previous execution of run_tests.py, remove any existing "run-x" test run folders + # and copy the main results folder contents into a new "run-0" folder to represent the previous run. Start the first + # run of this rerun as "run-1". + if args.rerun_failed_suites: + previous_report_file_path = f"{main_results_folder}{os.sep}output.xml" + if not os.path.exists(previous_report_file_path): + logger.error(f'No previous report file found at {previous_report_file_path} - unable to rerun failed suites') + sys.exit(1) + logger.info(f"Clearing old run folders prior to rerunning failed tests") + for old_run_folders in glob.glob(rf"{main_results_folder}{os.sep}run-*"): + shutil.rmtree(old_run_folders) + test_run_1_folder = f'{main_results_folder}{os.sep}run-0' + logger.info(f"Copying previous test results into new \"{test_run_1_folder}\" folder") + shutil.copytree(main_results_folder, test_run_1_folder) + else: + # Remove any existing test results if running from scratch. + if Path(main_results_folder).exists(): + shutil.rmtree(main_results_folder) + os.mkdir(main_results_folder) + + def run(): - robot_tests_folder = Path(__file__).absolute().parent - args = args_and_variables.initialise() + _setup_main_results_folder_for_first_run(args) + # Unzip any data files that may be used in tests. - unzip_data_files() + _unzip_data_files() # Upload test data files to storage if running locally. if args.env == "local": @@ -98,24 +117,19 @@ def run(): files_generator.create_public_release_files() files_generator.create_private_release_files() - install_chromedriver(args.chromedriver_version) + _install_chromedriver(args.chromedriver_version) - test_run_index = -1 - run_identifier_initial_value = create_run_identifier() + run_identifier_initial_value = _create_run_identifier() + + max_run_attempts = args.rerun_attempts + 1 + test_run_index = 0 - logger.info(f"Running Robot tests with {args.rerun_attempts} rerun attempts for any failing suites") + logger.info(f"Running Robot tests with {max_run_attempts} maximum run attempts") - if args.rerun_failed_suites: - logger.info(f"Clearing old run folders prior to rerunning failed tests") - for old_run_folders in glob.glob(rf"{main_results_folder}{os.sep}run-*"): - shutil.rmtree(old_run_folders) - try: # Run tests - while args.rerun_attempts is None or test_run_index < args.rerun_attempts: + while test_run_index < max_run_attempts: try: - test_run_index += 1 - # Ensure all SeleniumLibrary elements and keywords are updated to use a brand new # Selenium instance for every test (re)run. if test_run_index > 0: @@ -124,7 +138,7 @@ def run(): rerunning_failed_suites = args.rerun_failed_suites or test_run_index > 0 # Perform any cleanup before the test run. - clear_files_before_test_run(rerunning_failed_suites) + _clear_files_before_next_test_run_attempt(rerunning_failed_suites) # Create a folder to contain this test run attempt's outputs and reports. test_run_results_folder = f"{main_results_folder}{os.sep}run-{test_run_index + 1}" @@ -141,21 +155,16 @@ def run(): if args_and_variables.includes_data_changing_tests(args): admin_api.create_test_topic(run_identifier) - # If re-running failed suites, get the appropriate previous report file from which to determine which suites failed. - # If this is run 2 or more when using the `--rerun-attempts n` option, the path to the previous report will be in the - # previous run attempt's folder. - # If this is a rerun using the `--rerun-failed-suites` option, the path to the previous report will be in the main - # test results folder directly. + # If re-running failed suites, get the appropriate report file from the previous "run-x" folder. This will contain details of + # any failed tests from the previous run. if rerunning_failed_suites: - if test_run_index == 0: - previous_report_file = f"{robot_tests_folder}{os.sep}{main_results_folder}{os.sep}output.xml" - else: - previous_report_file = f"{robot_tests_folder}{os.sep}{main_results_folder}{os.sep}run-{test_run_index}{os.sep}output.xml" + previous_report_file = f"{main_results_folder}{os.sep}run-{test_run_index}{os.sep}output.xml" + logger.info(f"Using previous test run's results file \"{previous_report_file}\" to determine which failing suites to run") else: previous_report_file = None # Run the tests. - logger.info(f"Performing test run {test_run_index + 1} with unique identifier {run_identifier}") + logger.info(f"Performing test run {test_run_index + 1} in test run folder \"{test_run_results_folder}\" with unique identifier {run_identifier}") test_runners.execute_tests(args, test_run_results_folder, previous_report_file) if test_run_index > 0: @@ -167,6 +176,8 @@ def run(): logger.info("Tearing down test data...") admin_api.delete_test_topic() + test_run_index += 1 + # If all tests passed, return early. if not get_failing_test_suites(): break @@ -174,8 +185,9 @@ def run(): failing_suites = get_failing_test_suites() # Merge together all reports from all test runs. - number_of_test_runs = test_run_index + 1 - reports.merge_robot_reports(number_of_test_runs) + number_of_test_runs = test_run_index + first_run_folder_number = 0 if args.rerun_failed_suites else 1 + reports.merge_robot_reports(first_run_folder_number, number_of_test_runs) # Log the results of the merge test runs. reports.log_report_results(number_of_test_runs, failing_suites) @@ -196,7 +208,7 @@ def run(): current_dir = Path(__file__).absolute().parent os.chdir(current_dir) -setup_python_path() +_setup_python_path() # Run the tests! run() diff --git a/tests/robot-tests/test_runners.py b/tests/robot-tests/test_runners.py index 5f92496142c..b28f58f2680 100644 --- a/tests/robot-tests/test_runners.py +++ b/tests/robot-tests/test_runners.py @@ -43,7 +43,6 @@ def create_robot_arguments(arguments: argparse.Namespace, test_run_folder: str) # We want to add arguments on the first rerun attempt, but on subsequent attempts, we just want # to change rerunfailedsuites xml file we use robot_args += ["--output", "output.xml"] - return robot_args @@ -75,32 +74,29 @@ def _create_include_and_exclude_args(arguments: argparse.Namespace) -> []: def execute_tests(arguments: argparse.Namespace, test_run_folder: str, path_to_previous_report_file: str): robot_args = create_robot_arguments(arguments, test_run_folder) - - if arguments.interp == "robot": + if arguments.interp == "robot": if path_to_previous_report_file is not None: robot_args += ["--rerunfailedsuites", path_to_previous_report_file] - - robot_args += [arguments.tests] + robot_args += [arguments.tests] + + logger.info(f'Performing test run with Robot') robot.run_cli(robot_args, exit=False) elif arguments.interp == "pabot": - logger.info(f'Performing test run with Pabot ({arguments.processes} processes)') - + robot_args = ['--processes', arguments.processes] + robot_args + if path_to_previous_report_file is not None: - - logger.info(f'Re-running failed suites from {path_to_previous_report_file}') path_to_filtered_report_file = '_filtered.xml'.join(path_to_previous_report_file.rsplit('.xml', 1)) - reports.filter_out_passing_suites_from_report_file(path_to_previous_report_file, path_to_filtered_report_file) logger.info(f'Generated filtered report file containing only failing suites at {path_to_filtered_report_file}') - robot_args = ['--suitesfrom', path_to_filtered_report_file] + robot_args robot_args += [arguments.tests] + logger.info(f'Performing test run with Pabot ({arguments.processes} processes)') pabot.main_program(robot_args) From 3c1586b145d0a1cd7e929b95fceca2843f9fb036 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 8 Oct 2024 15:09:15 +0100 Subject: [PATCH 34/80] EES-5540 - adding additional test for just specifying the string "tests" for the tests to run --- tests/robot-tests/args_and_variables.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/robot-tests/args_and_variables.py b/tests/robot-tests/args_and_variables.py index 2723e9a93e8..6c512bcc64a 100644 --- a/tests/robot-tests/args_and_variables.py +++ b/tests/robot-tests/args_and_variables.py @@ -221,7 +221,8 @@ def validate_environment_variables(): # back slashes. def includes_data_changing_tests(arguments: argparse.Namespace): return ( - arguments.tests == "tests/" + arguments.tests == "tests" + or arguments.tests == "tests/" or arguments.tests == f"tests{os.sep}" or f"{os.sep}admin" in arguments.tests or "/admin" in arguments.tests From 48b5254213350c016c4f082b926d9368b4f1c685 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 8 Oct 2024 15:10:17 +0100 Subject: [PATCH 35/80] EES-5540 - specifying using "old" headless mode to overcome Windows issues when running headless with Chrome version 129 --- tests/robot-tests/tests/libs/common.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/robot-tests/tests/libs/common.robot b/tests/robot-tests/tests/libs/common.robot index 4ce02a7483e..dcf1f9bb059 100644 --- a/tests/robot-tests/tests/libs/common.robot +++ b/tests/robot-tests/tests/libs/common.robot @@ -63,7 +63,7 @@ user opens ie user opens chrome headlessly [Arguments] ${alias}=headless_chrome ${c_opts}= Evaluate sys.modules['selenium.webdriver'].ChromeOptions() sys, selenium.webdriver - Call Method ${c_opts} add_argument headless + Call Method ${c_opts} add_argument headless\=old Call Method ${c_opts} add_argument start-maximized Call Method ${c_opts} add_argument disable-extensions Call Method ${c_opts} add_argument disable-infobars From 31d03cec5ff5a49d49510183e88ccb77551c66bc Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 7 Oct 2024 14:17:10 +0100 Subject: [PATCH 36/80] EES-5526 - removed potentially redundant timezone code --- .../robot-tests/tests/public_api/public_api_preview_token.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/robot-tests/tests/public_api/public_api_preview_token.robot b/tests/robot-tests/tests/public_api/public_api_preview_token.robot index 71f74e9ea22..a139a38f657 100644 --- a/tests/robot-tests/tests/public_api/public_api_preview_token.robot +++ b/tests/robot-tests/tests/public_api/public_api_preview_token.robot @@ -240,7 +240,7 @@ User verifies the relevant fields on the active preview token page ${current_time_tomorrow}= get current local datetime %-I:%M %p 1 user checks page contains - ... The token expires: tomorrow at ${current_time_tomorrow} (local time) + ... The token expires: tomorrow at ${time_end} (local time) user checks page contains button Copy preview token user checks page contains button Revoke preview token From c52d421a0af0e2089116740f2d15dc730a005d8b Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 8 Oct 2024 11:43:29 +0100 Subject: [PATCH 37/80] EES-5526 - amending default timezone for test assertions to be Europe/London, and simplifying use of dates and datetimes by using more targeted keywords --- ...rove_release_as_publication_approver.robot | 8 +- ...lication_role_end_to_end_permissions.robot | 12 +- .../tests/admin/bau/adopt_methodology.robot | 10 +- .../bau/create_methodology_amendment.robot | 8 +- .../tests/admin/bau/prerelease.robot | 28 +-- .../admin/bau/prerelease_and_amend.robot | 24 +- .../tests/admin/bau/release_content.robot | 11 +- .../bau/prerelease_visibility.robot | 11 +- .../bau/publish_methodology.robot | 10 +- .../bau/publish_amend_and_cancel.robot | 14 +- .../bau/publish_release_and_amend.robot | 32 +-- .../bau/publish_release_and_amend_2.robot | 11 +- ..._tool_exclusions_by_geographic_level.robot | 2 +- .../robot-tests/tests/libs/admin-common.robot | 33 +-- tests/robot-tests/tests/libs/common.robot | 66 +++--- .../robot-tests/tests/libs/dates_and_times.py | 50 ++++ tests/robot-tests/tests/libs/utilities.py | 24 -- .../public_api/public_api_preview_token.robot | 221 +++++++++++------- .../public_api/public_api_restricted.robot | 177 ++++++++------ .../tests/seed_data/seed_data_common.robot | 6 +- 20 files changed, 423 insertions(+), 335 deletions(-) create mode 100644 tests/robot-tests/tests/libs/dates_and_times.py diff --git a/tests/robot-tests/tests/admin/analyst/edit_and_approve_release_as_publication_approver.robot b/tests/robot-tests/tests/admin/analyst/edit_and_approve_release_as_publication_approver.robot index 8d049d40b48..a8323bb766f 100644 --- a/tests/robot-tests/tests/admin/analyst/edit_and_approve_release_as_publication_approver.robot +++ b/tests/robot-tests/tests/admin/analyst/edit_and_approve_release_as_publication_approver.robot @@ -134,10 +134,10 @@ Put release back into draft Approve release for scheduled release ${days_until_release}= set variable 2 - ${publish_date_day}= get current datetime %-d ${days_until_release} - ${publish_date_month}= get current datetime %-m ${days_until_release} - ${publish_date_month_word}= get current datetime %B ${days_until_release} - ${publish_date_year}= get current datetime %Y ${days_until_release} + ${publish_date_day}= get london day of month offset_days=${days_until_release} + ${publish_date_month}= get london month date offset_days=${days_until_release} + ${publish_date_month_word}= get london month word offset_days=${days_until_release} + ${publish_date_year}= get london year offset_days=${days_until_release} user approves release for scheduled publication ... ${publish_date_day} diff --git a/tests/robot-tests/tests/admin/analyst/role_ui_permissions/release_and_publication_role_end_to_end_permissions.robot b/tests/robot-tests/tests/admin/analyst/role_ui_permissions/release_and_publication_role_end_to_end_permissions.robot index 498ab4293fc..6386e0408c3 100644 --- a/tests/robot-tests/tests/admin/analyst/role_ui_permissions/release_and_publication_role_end_to_end_permissions.robot +++ b/tests/robot-tests/tests/admin/analyst/role_ui_permissions/release_and_publication_role_end_to_end_permissions.robot @@ -103,8 +103,8 @@ Check publication owner can edit release status to "Ready for higher review" Validates Release status table is correct user waits until page contains element css:table user checks element count is x xpath://table/tbody/tr 1 - ${datetime} get current datetime %-d %B %Y - table cell should contain css:table 2 1 ${datetime} # Date + ${date} get london date + table cell should contain css:table 2 1 ${date} # Date table cell should contain css:table 2 2 HigherLevelReview # Status table cell should contain css:table 2 3 ready for higher review (publication owner) # Internal note table cell should contain css:table 2 4 1 # Release version @@ -116,10 +116,10 @@ Check publication owner can edit release status to "In draft" Validates Release status table is correct again user waits until page contains element css:table user checks element count is x xpath://table/tbody/tr 2 - ${datetime} get current datetime %-d %B %Y + ${date} get london date # New In draft row - table cell should contain css:table 2 1 ${datetime} # Date + table cell should contain css:table 2 1 ${date} # Date table cell should contain css:table 2 2 Draft # Status # Internal note table cell should contain css:table 2 3 Moving back to Draft state (publication owner) @@ -127,7 +127,7 @@ Validates Release status table is correct again table cell should contain css:table 2 5 ees-test.analyst1@education.gov.uk # By user # Higher review row - table cell should contain css:table 3 1 ${datetime} # Date + table cell should contain css:table 3 1 ${date} # Date table cell should contain css:table 3 2 HigherLevelReview # Status table cell should contain css:table 3 3 ready for higher review (publication owner) # Internal note table cell should contain css:table 3 4 1 # Release version @@ -224,7 +224,7 @@ Add a Footnote as a release approver Check release approver can create a release note user clicks link Content user adds a release note Test release note one - ${date} get current datetime %-d %B %Y + ${date} get london date user waits until element contains css:#release-notes li:nth-of-type(1) time ${date} user waits until element contains css:#release-notes li:nth-of-type(1) p Test release note one diff --git a/tests/robot-tests/tests/admin/bau/adopt_methodology.robot b/tests/robot-tests/tests/admin/bau/adopt_methodology.robot index a37c41439c0..cc2862fd3b0 100644 --- a/tests/robot-tests/tests/admin/bau/adopt_methodology.robot +++ b/tests/robot-tests/tests/admin/bau/adopt_methodology.robot @@ -63,7 +63,7 @@ Adopt a published Methodology # List of adoptable methodologies shows latest published version, not latest version - # that's why this is approved and not the draft amendment user checks element should contain ${selected_methodology_details} Approved - ${methodology_published_date}= get current datetime %-d %B %Y + ${methodology_published_date}= get london date user checks element should contain ${selected_methodology_details} ${methodology_published_date} user clicks button Save user waits until h2 is visible Manage methodologies @@ -83,10 +83,10 @@ Set methodology to published alongside release Schedule release to be published tomorrow user navigates to draft release page from dashboard ${ADOPTING_PUBLICATION_NAME} ${RELEASE_NAME} - ${day}= get current datetime %-d 1 - ${month}= get current datetime %-m 1 - ${month_word}= get current datetime %B 1 - ${year}= get current datetime %Y 1 + ${day}= get london day of month offset_days=1 + ${month}= get london month date offset_days=1 + ${month_word}= get london month word offset_days=1 + ${year}= get london year offset_days=1 user clicks link Sign off user clicks button Edit release status diff --git a/tests/robot-tests/tests/admin/bau/create_methodology_amendment.robot b/tests/robot-tests/tests/admin/bau/create_methodology_amendment.robot index d90d74364ce..b0394b71299 100644 --- a/tests/robot-tests/tests/admin/bau/create_methodology_amendment.robot +++ b/tests/robot-tests/tests/admin/bau/create_methodology_amendment.robot @@ -69,8 +69,8 @@ Create Methodology with some content and images user adds content to accordion section text block Methodology annex section 2 2 ... Adding Methodology annex 2 text block 2 ${METHODOLOGY_ANNEXES_EDITABLE_ACCORDION} user scrolls up 100 - user adds image to accordion section text block with retry Methodology annex section 2 2 test-infographic.png - ... Alt text for the uploaded annex image 3 ${METHODOLOGY_ANNEXES_EDITABLE_ACCORDION} + user adds image to accordion section text block with retry Methodology annex section 2 2 + ... test-infographic.png Alt text for the uploaded annex image 3 ${METHODOLOGY_ANNEXES_EDITABLE_ACCORDION} Verify the editable content is as expected user checks accordion section contains x blocks Methodology content section 1 1 @@ -106,7 +106,7 @@ Approve the Methodology user approves methodology for publication ${PUBLICATION_NAME} ${PUBLICATION_NAME} - first methodology version Verify the summary for the original Methodology is as expected - ${expected_published_date}= get current datetime %-d %B %Y + ${expected_published_date}= get london date user navigates to methodologies on publication page ... ${PUBLICATION_NAME} @@ -213,7 +213,7 @@ Visit the approved Amendment and check that its summary is as expected user clicks element xpath://*[text()="View"] ${ROW} user waits until h2 is visible Methodology summary - ${date}= get current datetime %-d %B %Y + ${date}= get london date user verifies methodology summary details ... ${PUBLICATION_NAME} ... ${PUBLICATION_NAME} diff --git a/tests/robot-tests/tests/admin/bau/prerelease.robot b/tests/robot-tests/tests/admin/bau/prerelease.robot index 70bf790c00e..d4b4797f7b8 100644 --- a/tests/robot-tests/tests/admin/bau/prerelease.robot +++ b/tests/robot-tests/tests/admin/bau/prerelease.robot @@ -105,10 +105,10 @@ Go to "Sign off" page user waits until page contains button Edit release status Approve release and wait for it to be Scheduled - ${day}= get current datetime %-d 2 - ${month}= get current datetime %-m 2 - ${month_word}= get current datetime %B 2 - ${year}= get current datetime %Y 2 + ${day}= get london day of month offset_days=2 + ${month}= get london month date offset_days=2 + ${month_word}= get london month word offset_days=2 + ${year}= get london year offset_days=2 user clicks button Edit release status user clicks radio Approved for publication @@ -142,11 +142,11 @@ Validate prerelease has not started user checks nth breadcrumb contains 1 Home user checks nth breadcrumb contains 2 Pre-release access - ${time_start}= get current london datetime %-d %B %Y at 00:00 1 - ${time_end}= get current london datetime %-d %B %Y 2 + ${start_date}= get london date offset_days=1 + ${end_date}= get london date offset_days=2 user checks page contains - ... Pre-release access will be available from ${time_start} until it is published on ${time_end}. + ... Pre-release access will be available from ${start_date} at 00:00 until it is published on ${end_date}. Go to prerelease access page user navigates to admin frontend ${RELEASE_URL}/prerelease-access @@ -247,18 +247,18 @@ Validate prerelease has not started for Analyst user user checks nth breadcrumb contains 1 Home user checks nth breadcrumb contains 2 Pre-release access - ${time_start}= get current london datetime %-d %B %Y at 00:00 1 - ${time_end}= get current london datetime %-d %B %Y 2 + ${start_date}= get london date offset_days=1 + ${end_date}= get london date offset_days=2 user checks page contains - ... Pre-release access will be available from ${time_start} until it is published on ${time_end}. + ... Pre-release access will be available from ${start_date} at 00:00 until it is published on ${end_date}. Start prerelease user changes to bau1 - ${day}= get current datetime %-d 1 - ${month}= get current datetime %-m 1 - ${month_word}= get current datetime %B 1 - ${year}= get current datetime %Y 1 + ${day}= get london day of month offset_days=1 + ${month}= get london month date offset_days=1 + ${month_word}= get london month word offset_days=1 + ${year}= get london year offset_days=1 user navigates to admin frontend ${RELEASE_URL}/status user clicks button Edit release status user enters text into element id:releaseStatusForm-publishScheduled-day ${day} diff --git a/tests/robot-tests/tests/admin/bau/prerelease_and_amend.robot b/tests/robot-tests/tests/admin/bau/prerelease_and_amend.robot index 92df2d946e8..2b42a1352e8 100644 --- a/tests/robot-tests/tests/admin/bau/prerelease_and_amend.robot +++ b/tests/robot-tests/tests/admin/bau/prerelease_and_amend.robot @@ -148,7 +148,7 @@ Add release note to amendment user clicks button Add note user enters text into element id:create-release-note-form-reason Test release note user clicks button Save note - ${date}= get current datetime ${DATE_FORMAT_MEDIUM} + ${date}= get london date user waits until element contains css:#release-notes li:nth-of-type(1) time ${date} user waits until element contains css:#release-notes li:nth-of-type(1) p Test release note @@ -205,10 +205,10 @@ Validate prerelease has not started for Analyst user during amendment as it is s Approve amendment for a scheduled release and check warning text user changes to bau1 - ${day}= get current datetime %-d 2 - ${month}= get current datetime %-m 2 - ${month_word}= get current datetime %B 2 - ${year}= get current datetime %Y 2 + ${day}= get london day of month offset_days=2 + ${month}= get london month date offset_days=2 + ${month_word}= get london month word offset_days=2 + ${year}= get london year offset_days=2 user navigates to admin frontend ${RELEASE_URL}/status user clicks button Edit release status user clicks radio Approved for publication @@ -239,18 +239,18 @@ Validate prerelease window is not yet open for Analyst user user checks nth breadcrumb contains 1 Home user checks nth breadcrumb contains 2 Pre-release access - ${time_start}= get current london datetime %-d %B %Y at 00:00 1 - ${time_end}= get current london datetime %-d %B %Y 2 + ${start_date}= get london date offset_days=1 + ${end_date}= get london date offset_days=2 user checks page contains - ... Pre-release access will be available from ${time_start} until it is published on ${time_end}. + ... Pre-release access will be available from ${start_date} at 00:00 until it is published on ${end_date}. Start prerelease user changes to bau1 - ${day}= get current datetime %-d 1 - ${month}= get current datetime %-m 1 - ${month_word}= get current datetime %B 1 - ${year}= get current datetime %Y 1 + ${day}= get london day of month offset_days=1 + ${month}= get london month date offset_days=1 + ${month_word}= get london month word offset_days=1 + ${year}= get london year offset_days=1 user navigates to admin frontend ${RELEASE_URL}/status user clicks button Edit release status user clicks radio On a specific date diff --git a/tests/robot-tests/tests/admin/bau/release_content.robot b/tests/robot-tests/tests/admin/bau/release_content.robot index f7a68a3ca1a..01b2d740a59 100644 --- a/tests/robot-tests/tests/admin/bau/release_content.robot +++ b/tests/robot-tests/tests/admin/bau/release_content.robot @@ -62,7 +62,7 @@ Add summary content to release Add release note to release user adds a release note Test release note one - ${date}= get current datetime %-d %B %Y + ${date}= get london date user waits until element contains css:#release-notes li:nth-of-type(1) time ${date} user waits until element contains css:#release-notes li:nth-of-type(1) p Test release note one @@ -81,9 +81,9 @@ Add secondary statistics ... ${expected_select_options} Check secondary statistics are included correctly - user waits until element is visible id:${SECONDARY_STATS_TABLE_TAB_ID} %{WAIT_MEDIUM} + user waits until element is visible id:${SECONDARY_STATS_TABLE_TAB_ID} %{WAIT_MEDIUM} user scrolls to element id:${SECONDARY_STATS_TABLE_TAB_ID} - user clicks element id:${SECONDARY_STATS_TABLE_TAB_ID} + user clicks element id:${SECONDARY_STATS_TABLE_TAB_ID} user checks page contains Data Block 1 title user checks page contains element css:table user checks page contains button Change secondary stats @@ -249,8 +249,7 @@ Verify that validation prevents adding an invalid link *** Keywords *** - User waits until secondary stats table tab is visible [Arguments] ${SECONDARY_STATS_TABLE_TAB_ID} ${timeout}= %{TIMEOUT} - wait until keyword succeeds ${timeout} 5 sec Element Should Be Visible id=${SECONDARY_STATS_TABLE_TAB_ID} - + wait until keyword succeeds ${timeout} 5 sec Element Should Be Visible + ... id=${SECONDARY_STATS_TABLE_TAB_ID} diff --git a/tests/robot-tests/tests/admin_and_public/bau/prerelease_visibility.robot b/tests/robot-tests/tests/admin_and_public/bau/prerelease_visibility.robot index deae62cf679..677f9b93136 100644 --- a/tests/robot-tests/tests/admin_and_public/bau/prerelease_visibility.robot +++ b/tests/robot-tests/tests/admin_and_public/bau/prerelease_visibility.robot @@ -28,7 +28,8 @@ Verify release summary ... Accredited official statistics Upload subject - user uploads subject and waits until complete UI test subject upload-file-test.csv upload-file-test.meta.csv + user uploads subject and waits until complete UI test subject upload-file-test.csv + ... upload-file-test.meta.csv Check release isn't publically visible user clicks link Sign off @@ -103,10 +104,10 @@ Go to "Sign off" page user waits until page contains button Edit release status %{WAIT_SMALL} Approve release and wait for it to be Scheduled - ${day}= get current datetime %-d 2 - ${month}= get current datetime %-m 2 - ${month_word}= get current datetime %B 2 - ${year}= get current datetime %Y 2 + ${day}= get london day of month offset_days=2 + ${month}= get london month date offset_days=2 + ${month_word}= get london month word offset_days=2 + ${year}= get london year offset_days=2 user clicks button Edit release status user clicks radio Approved for publication diff --git a/tests/robot-tests/tests/admin_and_public/bau/publish_methodology.robot b/tests/robot-tests/tests/admin_and_public/bau/publish_methodology.robot index 097e5a25b1d..2380035eed0 100644 --- a/tests/robot-tests/tests/admin_and_public/bau/publish_methodology.robot +++ b/tests/robot-tests/tests/admin_and_public/bau/publish_methodology.robot @@ -151,8 +151,8 @@ Verify that the methodology 'Published' tag and datetime is shown ${ROW}= user gets table row ${PUBLICATION_NAME} user checks element contains ${ROW} Published - ${DATE}= get current datetime %-d %B %Y - user checks element contains ${ROW} ${DATE} + ${date}= get london date + user checks element contains ${ROW} ${date} Verify that the methodology is visible on the public methodologies page with the expected URL user navigates to public methodologies page @@ -198,7 +198,7 @@ Verify that the methodology displays a link to the publication ... css:[aria-labelledby="related-information"] Verify that the methodology content is correct - ${date}= get current datetime %-d %B %Y + ${date}= get london date user checks summary list contains Published ${date} user checks accordion is in position Methodology content section 1 1 id:content @@ -363,7 +363,7 @@ Verify that the amended methodology displays a link to the publication ... css:[aria-labelledby="related-information"] Verify that the amended methodology content is correct - ${date}= get current datetime %-d %B %Y + ${date}= get london date user checks summary list contains Published ${date} user checks summary list contains Last updated ${date} @@ -400,7 +400,7 @@ Verify that the amended methodology content is correct user checks element contains ${annexe_section_1} Annexe 1 Verify the list of notes - ${date}= get current datetime %-d %B %Y + ${date}= get london date user opens details dropdown See all notes (2) user waits until page contains element css:[data-testid="notes"] li limit=2 user checks methodology note 1 ${date} Latest note diff --git a/tests/robot-tests/tests/admin_and_public_2/bau/publish_amend_and_cancel.robot b/tests/robot-tests/tests/admin_and_public_2/bau/publish_amend_and_cancel.robot index 4ecbcea66aa..9583f4db208 100644 --- a/tests/robot-tests/tests/admin_and_public_2/bau/publish_amend_and_cancel.robot +++ b/tests/robot-tests/tests/admin_and_public_2/bau/publish_amend_and_cancel.robot @@ -152,10 +152,10 @@ Add public prerelease access list Approve release for scheduled publication ${days_until_release}= set variable 0 - ${publish_date_day}= get current datetime %-d ${days_until_release} - ${publish_date_month}= get current datetime %-m ${days_until_release} - ${publish_date_month_word}= get current datetime %B ${days_until_release} - ${publish_date_year}= get current datetime %Y ${days_until_release} + ${publish_date_day}= get london day of month offset_days=${days_until_release} + ${publish_date_month}= get london month date offset_days=${days_until_release} + ${publish_date_month_word}= get london month word offset_days=${days_until_release} + ${publish_date_year}= get london year offset_days=${days_until_release} user approves release for scheduled publication ... ${publish_date_day} @@ -170,9 +170,9 @@ Approve release for scheduled publication Publish the scheduled release user waits for scheduled release to be published immediately - ${publish_date_day}= get current datetime %-d - ${publish_date_month_word}= get current datetime %B - ${publish_date_year}= get current datetime %Y + ${publish_date_day}= get london day of month + ${publish_date_month_word}= get london month word + ${publish_date_year}= get london year set suite variable ${EXPECTED_PUBLISHED_DATE} ... ${publish_date_day} ${publish_date_month_word} ${publish_date_year} diff --git a/tests/robot-tests/tests/admin_and_public_2/bau/publish_release_and_amend.robot b/tests/robot-tests/tests/admin_and_public_2/bau/publish_release_and_amend.robot index f0e6d99fca3..d933eda5c03 100644 --- a/tests/robot-tests/tests/admin_and_public_2/bau/publish_release_and_amend.robot +++ b/tests/robot-tests/tests/admin_and_public_2/bau/publish_release_and_amend.robot @@ -244,10 +244,10 @@ Add public prerelease access list Approve release for scheduled publication ${days_until_release}= set variable 0 - ${publish_date_day}= get current datetime %-d ${days_until_release} - ${publish_date_month}= get current datetime %-m ${days_until_release} - ${publish_date_month_word}= get current datetime %B ${days_until_release} - ${publish_date_year}= get current datetime %Y ${days_until_release} + ${publish_date_day}= get london day of month offset_days=${days_until_release} + ${publish_date_month}= get london month date offset_days=${days_until_release} + ${publish_date_month_word}= get london month word offset_days=${days_until_release} + ${publish_date_year}= get london year offset_days=${days_until_release} user approves release for scheduled publication ... ${publish_date_day} @@ -273,7 +273,7 @@ Get public release link Publish the scheduled release user waits for scheduled release to be published immediately - ${EXPECTED_PUBLISHED_DATE}= get current datetime ${DATE_FORMAT_MEDIUM} + ${EXPECTED_PUBLISHED_DATE}= get london date set suite variable ${EXPECTED_PUBLISHED_DATE} Verify newly published release is on Find Statistics page @@ -724,7 +724,7 @@ Add release note to first amendment user clicks button Add note user enters text into element id:create-release-note-form-reason Test release note one user clicks button Save note - ${date}= get current datetime ${DATE_FORMAT_MEDIUM} + ${date}= get london date user waits until element contains css:#release-notes li:nth-of-type(1) time ${date} user waits until element contains css:#release-notes li:nth-of-type(1) p Test release note one @@ -739,10 +739,10 @@ Update public prerelease access list Approve amendment for scheduled release ${days_until_release}= set variable 1 - ${publish_date_day}= get current datetime %-d ${days_until_release} - ${publish_date_month}= get current datetime %-m ${days_until_release} - ${publish_date_month_word}= get current datetime %B ${days_until_release} - ${publish_date_year}= get current datetime %Y ${days_until_release} + ${publish_date_day}= get london day of month offset_days=${days_until_release} + ${publish_date_month}= get london month date offset_days=${days_until_release} + ${publish_date_month_word}= get london month word offset_days=${days_until_release} + ${publish_date_year}= get london year offset_days=${days_until_release} user approves release for scheduled publication ... ${publish_date_day} @@ -753,7 +753,7 @@ Approve amendment for scheduled release user waits for scheduled release to be published immediately - ${EXPECTED_PUBLISHED_DATE}= get current datetime ${DATE_FORMAT_MEDIUM} + ${EXPECTED_PUBLISHED_DATE}= get london date set suite variable ${EXPECTED_PUBLISHED_DATE} Verify amendment is on Find Statistics page again @@ -966,7 +966,7 @@ Override release published date to past date ${release_id}= get release id from url ${published_override}= Get Current Date UTC increment=-1000 days result_format=datetime user updates release published date via api ${release_id} ${published_override} - ${EXPECTED_PUBLISHED_DATE}= format datetime ${published_override} ${DATE_FORMAT_MEDIUM} + ${EXPECTED_PUBLISHED_DATE}= get london date offset_days=-1000 set suite variable ${EXPECTED_PUBLISHED_DATE} Verify published date on publication page is overriden with past date @@ -1005,9 +1005,9 @@ Remove the content section that originally contained the deleted data block Approve release amendment for scheduled publication and update published date ${days_until_release}= set variable 2 - ${publish_date_day}= get current datetime %-d ${days_until_release} - ${publish_date_month}= get current datetime %-m ${days_until_release} - ${publish_date_year}= get current datetime %Y ${days_until_release} + ${publish_date_day}= get london day of month offset_days=${days_until_release} + ${publish_date_month}= get london month date offset_days=${days_until_release} + ${publish_date_year}= get london year offset_days=${days_until_release} user approves release for scheduled publication ... ${publish_date_day} ... ${publish_date_month} @@ -1016,7 +1016,7 @@ Approve release amendment for scheduled publication and update published date ... next_release_year=4001 ... update_amendment_published_date=${True} user waits for scheduled release to be published immediately - ${EXPECTED_PUBLISHED_DATE}= get current datetime ${DATE_FORMAT_MEDIUM} + ${EXPECTED_PUBLISHED_DATE}= get london date set suite variable ${EXPECTED_PUBLISHED_DATE} Verify published date on publication page has been updated diff --git a/tests/robot-tests/tests/admin_and_public_2/bau/publish_release_and_amend_2.robot b/tests/robot-tests/tests/admin_and_public_2/bau/publish_release_and_amend_2.robot index 28bf318b597..90731c59b50 100644 --- a/tests/robot-tests/tests/admin_and_public_2/bau/publish_release_and_amend_2.robot +++ b/tests/robot-tests/tests/admin_and_public_2/bau/publish_release_and_amend_2.robot @@ -33,7 +33,8 @@ Create new release Upload another subject (for deletion later) user waits until page contains element id:dataFileUploadForm-subjectTitle - user uploads subject and waits until complete ${SECOND_SUBJECT} upload-file-test.csv upload-file-test.meta.csv + user uploads subject and waits until complete ${SECOND_SUBJECT} upload-file-test.csv + ... upload-file-test.meta.csv Add data guidance to subject user clicks link Data guidance @@ -292,7 +293,7 @@ Add release note to release amendment user clicks button Add note user enters text into element id:create-release-note-form-reason Test release note one user clicks button Save note - ${date} get current datetime %-d %B %Y + ${date} get london date user waits until element contains css:#release-notes li:nth-of-type(1) time ${date} user waits until element contains css:#release-notes li:nth-of-type(1) p Test release note one @@ -303,8 +304,8 @@ Go to "Sign off" page Validate Release status table row is correct user waits until page contains element css:table user checks element count is x xpath://table/tbody/tr 1 - ${datetime} get current datetime %-d %B %Y - table cell should contain css:table 2 1 ${datetime} # Date + ${date} get london date + table cell should contain css:table 2 1 ${date} # Date table cell should contain css:table 2 2 Approved # Status table cell should contain css:table 2 3 Approved by UI tests # Internal note table cell should contain css:table 2 4 1 # Release version @@ -425,7 +426,7 @@ Update Seven filters footnote Add release note for new release amendment user clicks link Content user adds a release note Test release note two - ${date} get current datetime %-d %B %Y + ${date} get london date user waits until element contains css:#release-notes li:nth-of-type(1) time ${date} user waits until element contains css:#release-notes li:nth-of-type(1) p Test release note two diff --git a/tests/robot-tests/tests/general_public/table_tool_exclusions_by_geographic_level.robot b/tests/robot-tests/tests/general_public/table_tool_exclusions_by_geographic_level.robot index 88061fdd162..13bcd02da3f 100644 --- a/tests/robot-tests/tests/general_public/table_tool_exclusions_by_geographic_level.robot +++ b/tests/robot-tests/tests/general_public/table_tool_exclusions_by_geographic_level.robot @@ -161,7 +161,7 @@ User validates permanent link works correctly ... 'Exclusions by geographic level' from '${EXCLUSIONS_PUBLICATION_TITLE}' User validates permalink contains correct date - ${date}= get current datetime %-d %B %Y + ${date}= get london date user checks page contains element xpath://*[@data-testid="created-date"]//strong//time[text()="${date}"] User validates permalink table headers diff --git a/tests/robot-tests/tests/libs/admin-common.robot b/tests/robot-tests/tests/libs/admin-common.robot index bd44e72a706..b459ac6a390 100644 --- a/tests/robot-tests/tests/libs/admin-common.robot +++ b/tests/robot-tests/tests/libs/admin-common.robot @@ -407,7 +407,7 @@ user adds note to methodology user clicks button Add note user enters text into element label:New methodology note ${note} user clicks button Save note - ${date}= get current datetime %-d %B %Y + ${date}= get london date user waits until element contains css:#methodologyNotes time ${date} user waits until element contains css:#methodologyNotes p ${note} @@ -569,11 +569,11 @@ user uploads subject and waits until complete ... ${META_FILE} ... ${FOLDER}=${FILES_DIR} user uploads subject - ... ${SUBJECT_NAME} - ... ${SUBJECT_FILE} - ... ${META_FILE} - ... Complete - ... ${FOLDER} + ... ${SUBJECT_NAME} + ... ${SUBJECT_FILE} + ... ${META_FILE} + ... Complete + ... ${FOLDER} user uploads subject and waits until importing [Arguments] @@ -582,12 +582,12 @@ user uploads subject and waits until importing ... ${META_FILE} ... ${FOLDER}=${FILES_DIR} user uploads subject - ... ${SUBJECT_NAME} - ... ${SUBJECT_FILE} - ... ${META_FILE} - ... Importing - ... ${FOLDER} - + ... ${SUBJECT_NAME} + ... ${SUBJECT_FILE} + ... ${META_FILE} + ... Importing + ... ${FOLDER} + user uploads subject [Arguments] ... ${SUBJECT_NAME} @@ -603,14 +603,14 @@ user uploads subject user clicks button Upload data files user waits until h2 is visible Uploaded data files %{WAIT_LONG} user waits until page contains accordion section ${SUBJECT_NAME} %{WAIT_SMALL} - user scrolls to accordion section ${SUBJECT_NAME} + user scrolls to accordion section ${SUBJECT_NAME} user opens accordion section ${SUBJECT_NAME} ${section}= user gets accordion section content element ${SUBJECT_NAME} - + IF "${IMPORT_STATUS}" != "Importing" user waits until page finishes loading END - + user checks headed table body row contains Status ${IMPORT_STATUS} ${section} %{WAIT_DATA_FILE_IMPORT} user waits until data upload is completed @@ -941,7 +941,8 @@ user updates free text key stat user enters text into element xpath://*[@data-testid="keyStat"][${tile_num}]//input[@name="guidanceTitle"] ... ${guidance_title} - user enters text into element xpath://*[@data-testid="keyStat"][${tile_num}]//textarea[@name="guidanceText"] ${guidance_text} + user enters text into element xpath://*[@data-testid="keyStat"][${tile_num}]//textarea[@name="guidanceText"] + ... ${guidance_text} user clicks button Save user waits until page does not contain button Save diff --git a/tests/robot-tests/tests/libs/common.robot b/tests/robot-tests/tests/libs/common.robot index dcf1f9bb059..5b9cf9d70f3 100644 --- a/tests/robot-tests/tests/libs/common.robot +++ b/tests/robot-tests/tests/libs/common.robot @@ -2,9 +2,10 @@ Library SeleniumLibrary timeout=%{TIMEOUT} implicit_wait=%{IMPLICIT_WAIT} run_on_failure=record test failure Library OperatingSystem Library Collections +Library dates_and_times.py +Library fail_fast.py Library file_operations.py Library utilities.py -Library fail_fast.py Library visual.py Resource ./tables-common.robot Resource ./table_tool.robot @@ -20,7 +21,6 @@ ${DOWNLOADS_DIR}= ${EXECDIR}${/}test-results${/}downloads$ ${timeout}= %{TIMEOUT} ${implicit_wait}= %{IMPLICIT_WAIT} ${prompt_to_continue_on_failure}= 0 -${DATE_FORMAT_MEDIUM}= %-d %B %Y *** Keywords *** @@ -454,7 +454,7 @@ user checks element should not contain user checks input field contains [Arguments] ${element} ${text} page should contain textfield ${element} - textfield should contain ${element} ${text} + textfield should contain ${element} ${text} user checks page contains [Arguments] ${text} @@ -489,7 +489,7 @@ user clicks link by index [Arguments] ${text} ${index}=1 ${parent}=css:body ${xpath}= set variable (//a[text()='${text}'])[${index}] ${button}= get webelement ${xpath} - user clicks element ${button} ${parent} + user clicks element ${button} ${parent} user clicks link by visible text [Arguments] ${text} ${parent}=css:body @@ -508,7 +508,7 @@ user clicks button by index [Arguments] ${text} ${index}=1 ${parent}=css:body ${xpath}= set variable (//button[text()='${text}'])[${index}] ${button}= get webelement ${xpath} - user clicks element ${button} ${parent} + user clicks element ${button} ${parent} user waits until button is clickable [Arguments] ${button_text} @@ -707,9 +707,9 @@ user checks url equals user checks url without auth equals [Arguments] ${expected} ${current_url}= get location - ${remove_auth_current_url}= remove auth from url ${current_url} + ${remove_auth_current_url}= remove auth from url ${current_url} set variable ${remove_auth_current_url} - should contain ${remove_auth_current_url} ${expected} + should contain ${remove_auth_current_url} ${expected} user checks page contains link [Arguments] @@ -832,7 +832,7 @@ user clicks checkbox user clicks checkbox by selector [Arguments] ${locator} - user scrolls to element ${locator} + user scrolls to element ${locator} user clicks element ${locator} user checks checkbox is checked @@ -1019,35 +1019,29 @@ user waits for caches to expire sleep %{WAIT_CACHE_EXPIRY} user wait for option to be available and select it - [Arguments] ${dropdown_locator} ${option_text} ${timeout}=%{TIMEOUT} - wait until keyword succeeds ${timeout} 1s check option exist in dropdown ${dropdown_locator} ${option_text} - select from list by label ${dropdown_locator} ${option_text} + [Arguments] ${dropdown_locator} ${option_text} ${timeout}=%{TIMEOUT} + wait until keyword succeeds ${timeout} 1s check option exist in dropdown ${dropdown_locator} + ... ${option_text} + select from list by label ${dropdown_locator} ${option_text} check option exist in dropdown - [Arguments] ${dropdown_locator} ${option_text} - ${options}= get webelements ${dropdown_locator} > option - ${all_texts}= Create List - - FOR ${option} IN @{options} - ${text}= get text ${option} - Append To List ${all_texts} ${text} - END - # Adding logging to help catch intermittent test failures - Log to console \n\tAll Texts: ${all_texts} - ${matched}= Run Keyword And Return Status Should Contain ${all_texts} ${option_text} - - IF ${matched} - # Adding logging to help catch intermittent test failures - Log to console \n\tOption '${option_text}' found in the dropdown. - ELSE - # Adding logging to help catch intermittent test failures - Log to console \n\tOption '${option_text}' not found in the dropdown. - END - Return From Keyword ${matched} - - - - - + [Arguments] ${dropdown_locator} ${option_text} + ${options}= get webelements ${dropdown_locator} > option + ${all_texts}= Create List + FOR ${option} IN @{options} + ${text}= get text ${option} + Append To List ${all_texts} ${text} + END + # Adding logging to help catch intermittent test failures + Log to console \n\tAll Texts: ${all_texts} + ${matched}= Run Keyword And Return Status Should Contain ${all_texts} ${option_text} + IF ${matched} + # Adding logging to help catch intermittent test failures + Log to console \n\tOption '${option_text}' found in the dropdown. + ELSE + # Adding logging to help catch intermittent test failures + Log to console \n\tOption '${option_text}' not found in the dropdown. + END + Return From Keyword ${matched} diff --git a/tests/robot-tests/tests/libs/dates_and_times.py b/tests/robot-tests/tests/libs/dates_and_times.py new file mode 100644 index 00000000000..22ff7ef7128 --- /dev/null +++ b/tests/robot-tests/tests/libs/dates_and_times.py @@ -0,0 +1,50 @@ +import datetime +import os + +import pytz +from tests.libs.selenium_elements import sl + + +def get_london_date(offset_days: int = 0, format_string: str = "%-d %B %Y") -> str: + return _get_date_and_time(offset_days, format_string, "Europe/London") + + +def get_london_date_and_time(offset_days: int = 0, format_string: str = "%-d %B %Y") -> str: + return _get_date_and_time(offset_days, format_string, "Europe/London") + + +def get_local_browser_date_and_time(offset_days: int = 0, format_string: str = "%-d %B %Y") -> str: + return _get_date_and_time(offset_days, format_string, _get_browser_timezone()) + + +def get_london_day_of_month(offset_days: int = 0) -> str: + return get_london_date_and_time(offset_days, "%-d") + + +def get_london_month_date(offset_days: int = 0) -> str: + return get_london_date_and_time(offset_days, "%-m") + + +def get_london_month_word(offset_days: int = 0) -> str: + return get_london_date_and_time(offset_days, "%B") + + +def get_london_year(offset_days: int = 0) -> str: + return get_london_date_and_time(offset_days, "%Y") + + +def _get_browser_timezone(): + return sl().driver.execute_script("return Intl.DateTimeFormat().resolvedOptions().timeZone;") + + +def _get_date_and_time(offset_days: int, format_string: str, timezone: str) -> str: + return _format_datetime( + datetime.datetime.now(pytz.timezone(timezone)) + datetime.timedelta(days=offset_days), format_string + ) + + +def _format_datetime(datetime: datetime, format_string: str) -> str: + if os.name == "nt": + format_string = format_string.replace("%-", "%#") + + return datetime.strftime(format_string) diff --git a/tests/robot-tests/tests/libs/utilities.py b/tests/robot-tests/tests/libs/utilities.py index ed25043690a..11ff15e68cc 100644 --- a/tests/robot-tests/tests/libs/utilities.py +++ b/tests/robot-tests/tests/libs/utilities.py @@ -7,7 +7,6 @@ from typing import Union from urllib.parse import urlparse, urlunparse -import pytz import utilities_init import visual from robot.libraries.BuiltIn import BuiltIn @@ -280,25 +279,6 @@ def set_cookie_from_json(cookie_json): sl().driver.add_cookie(cookie_dict) -def get_current_datetime(strf: str, offset_days: int = 0, timezone: str = "UTC") -> str: - return format_datetime(datetime.datetime.now(pytz.timezone(timezone)) + datetime.timedelta(days=offset_days), strf) - - -def get_current_london_datetime(strf: str, offset_days: int = 0) -> str: - return get_current_datetime(strf, offset_days, "Europe/London") - - -def get_current_local_datetime(strf: str, offset_days: int = 0) -> str: - return get_current_datetime(strf, offset_days, _get_browser_timezone()) - - -def format_datetime(datetime: datetime, strf: str) -> str: - if os.name == "nt": - strf = strf.replace("%-", "%#") - - return datetime.strftime(strf) - - def user_should_be_at_top_of_page(): (x, y) = sl().get_window_position() if y != 0: @@ -438,7 +418,3 @@ def get_child_element_with_retry(parent_locator: object, child_locator: str, max logger.warn(f"Child element not found, after ({max_retries}) retries") time.sleep(retry_delay) raise AssertionError(f"Failed to find child element after {max_retries} retries.") - - -def _get_browser_timezone(): - return sl().driver.execute_script("return Intl.DateTimeFormat().resolvedOptions().timeZone;") diff --git a/tests/robot-tests/tests/public_api/public_api_preview_token.robot b/tests/robot-tests/tests/public_api/public_api_preview_token.robot index a139a38f657..29b65cb3eef 100644 --- a/tests/robot-tests/tests/public_api/public_api_preview_token.robot +++ b/tests/robot-tests/tests/public_api/public_api_preview_token.robot @@ -11,13 +11,15 @@ Suite Setup user signs in as bau1 Suite Teardown user closes the browser Test Setup fail test fast if required + *** Variables *** -${PUBLICATION_NAME}= UI tests - Public API - preview token %{RUN_IDENTIFIER} -${RELEASE_NAME}= Academic year Q1 -${ACADEMIC_YEAR}= 3000 -${SUBJECT_NAME_1}= UI test subject 1 -${SUBJECT_NAME_2}= UI test subject 2 -${PREVIEW_TOKEN_NAME}= Test token +${PUBLICATION_NAME}= UI tests - Public API - preview token %{RUN_IDENTIFIER} +${RELEASE_NAME}= Academic year Q1 +${ACADEMIC_YEAR}= 3000 +${SUBJECT_NAME_1}= UI test subject 1 +${SUBJECT_NAME_2}= UI test subject 2 +${PREVIEW_TOKEN_NAME}= Test token + *** Test Cases *** Create publication and release @@ -33,8 +35,10 @@ Verify release summary user verifies release summary Academic year Q1 3000/01 Accredited official statistics Upload data files - user uploads subject and waits until complete ${SUBJECT_NAME_1} seven_filters.csv seven_filters.meta.csv ${PUBLIC_API_FILES_DIR} - user uploads subject and waits until complete ${SUBJECT_NAME_2} tiny-two-filters.csv tiny-two-filters.meta.csv ${PUBLIC_API_FILES_DIR} + user uploads subject and waits until complete ${SUBJECT_NAME_1} seven_filters.csv seven_filters.meta.csv + ... ${PUBLIC_API_FILES_DIR} + user uploads subject and waits until complete ${SUBJECT_NAME_2} tiny-two-filters.csv + ... tiny-two-filters.meta.csv ${PUBLIC_API_FILES_DIR} Add data guidance to subjects user clicks link Data and files @@ -62,7 +66,7 @@ Create 1st API dataset user clicks button Create API data set ${modal}= user waits until modal is visible Create a new API data set - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_1} + user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_1} user clicks button Confirm new API data set user waits until page finishes loading @@ -76,7 +80,7 @@ Create 2nd API dataset user clicks link Back to API data sets user clicks button Create API data set ${modal}= user waits until modal is visible Create a new API data set - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_2} + user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_2} user clicks button Confirm new API data set user waits until page finishes loading @@ -90,10 +94,11 @@ Verify the contents inside the 'Draft API datasets' table user clicks link Back to API data sets user waits until h3 is visible Draft API data sets - user checks table column heading contains 1 1 Draft version xpath://table[@data-testid='draft-api-data-sets'] - user checks table column heading contains 1 2 Name xpath://table[@data-testid='draft-api-data-sets'] - user checks table column heading contains 1 3 Status xpath://table[@data-testid='draft-api-data-sets'] - user checks table column heading contains 1 4 Actions xpath://table[@data-testid='draft-api-data-sets'] + user checks table column heading contains 1 1 Draft version + ... xpath://table[@data-testid='draft-api-data-sets'] + user checks table column heading contains 1 2 Name xpath://table[@data-testid='draft-api-data-sets'] + user checks table column heading contains 1 3 Status xpath://table[@data-testid='draft-api-data-sets'] + user checks table column heading contains 1 4 Actions xpath://table[@data-testid='draft-api-data-sets'] user checks table cell contains 1 1 v1.0 xpath://table[@data-testid='draft-api-data-sets'] user checks table cell contains 1 3 Ready xpath://table[@data-testid='draft-api-data-sets'] @@ -107,46 +112,67 @@ Click on 'View Details' link(First API dataset) user checks table headings for Draft version details table User checks row data contents inside the 'Draft API datasets' summary table - user checks contents inside the cell value v1.0 xpath://dl[@data-testid="draft-version-summary"]/div/dd[@data-testid='Version-value']/strong - user checks contents inside the cell value Ready xpath:(//div[@data-testid="Status"]//dd[@data-testid="Status-value"]//strong)[2] - user checks contents inside the cell value Academic year Q1 3000/01 xpath:(//div[@data-testid="Release"]//dd[@data-testid="Release-value"]//a)[1] - user checks contents inside the cell value ${SUBJECT_NAME_1} xpath://div[@data-testid="Data set file"]//dd[@data-testid="Data set file-value"] - user checks contents inside the cell value National xpath://div[@data-testid="Geographic levels"]//dd[@data-testid="Geographic levels-value"] - user checks contents inside the cell value 2012/13 xpath://div[@data-testid="Time periods"]//dd[@data-testid="Time periods-value"] - - user checks contents inside the cell value Lower quartile annualised earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[1] - user checks contents inside the cell value Median annualised earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[2] - user checks contents inside the cell value Number of learners with earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[3] - - user clicks button Show 1 more indicator xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"] - - user checks contents inside the cell value Upper quartile annualised earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[4] - - user checks contents inside the cell value Cheese xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[1] - user checks contents inside the cell value Colour xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[2] - user checks contents inside the cell value Ethnicity group xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[3] - - user clicks button Show 4 more filters xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"] - - user checks contents inside the cell value Gender xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[4] - user checks contents inside the cell value Level of learning xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[5] - user checks contents inside the cell value Number of years after achievement of learning aim xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[6] - user checks contents inside the cell value Provision xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[7] - - user checks contents inside the cell value Preview API data set xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[1]/a - user checks contents inside the cell value View preview token log xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[2]/a - user checks contents inside the cell value Remove draft version xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[3]/button + user checks contents inside the cell value v1.0 + ... xpath://dl[@data-testid="draft-version-summary"]/div/dd[@data-testid='Version-value']/strong + user checks contents inside the cell value Ready + ... xpath:(//div[@data-testid="Status"]//dd[@data-testid="Status-value"]//strong)[2] + user checks contents inside the cell value Academic year Q1 3000/01 + ... xpath:(//div[@data-testid="Release"]//dd[@data-testid="Release-value"]//a)[1] + user checks contents inside the cell value ${SUBJECT_NAME_1} + ... xpath://div[@data-testid="Data set file"]//dd[@data-testid="Data set file-value"] + user checks contents inside the cell value National + ... xpath://div[@data-testid="Geographic levels"]//dd[@data-testid="Geographic levels-value"] + user checks contents inside the cell value 2012/13 + ... xpath://div[@data-testid="Time periods"]//dd[@data-testid="Time periods-value"] + + user checks contents inside the cell value Lower quartile annualised earnings + ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[1] + user checks contents inside the cell value Median annualised earnings + ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[2] + user checks contents inside the cell value Number of learners with earnings + ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[3] + + user clicks button Show 1 more indicator + ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"] + + user checks contents inside the cell value Upper quartile annualised earnings + ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[4] + + user checks contents inside the cell value Cheese + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[1] + user checks contents inside the cell value Colour + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[2] + user checks contents inside the cell value Ethnicity group + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[3] + + user clicks button Show 4 more filters xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"] + + user checks contents inside the cell value Gender + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[4] + user checks contents inside the cell value Level of learning + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[5] + user checks contents inside the cell value Number of years after achievement of learning aim + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[6] + user checks contents inside the cell value Provision + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[7] + + user checks contents inside the cell value Preview API data set + ... xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[1]/a + user checks contents inside the cell value View preview token log + ... xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[2]/a + user checks contents inside the cell value Remove draft version + ... xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[3]/button User clicks on 'Preview API data set' link user clicks link containing text Preview API data set User clicks on 'Generate preview token' - user clicks button Generate preview token + user clicks button Generate preview token User creates preview token through 'Generate preview token' modal window ${modal}= user waits until modal is visible Generate preview token user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] ${PREVIEW_TOKEN_NAME} - user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] + user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] user clicks button Continue user waits until page finishes loading @@ -163,12 +189,12 @@ User revokes preview token user waits until page contains Generate API data set preview token User again clicks on 'Generate preview token' - user clicks button Generate preview token + user clicks button Generate preview token User creates another preview token through 'Generate preview token' modal window ${modal}= user waits until modal is visible Generate preview token user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] ${PREVIEW_TOKEN_NAME} - user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] + user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] user clicks button Continue user waits until page finishes loading @@ -188,11 +214,11 @@ User cancels revoking preview token User cancels creating preview token user clicks link Back to API data set details user clicks link containing text Preview API data set - user clicks button Generate preview token + user clicks button Generate preview token ${modal}= user waits until modal is visible Generate preview token user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] ${PREVIEW_TOKEN_NAME} - user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] + user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] user clicks button Cancel user waits until page finishes loading @@ -228,7 +254,7 @@ User verifies the relevant fields on the active preview token page ${modal}= user waits until modal is visible Generate preview token user enters text into element css:input[id="apiDataSetTokenCreateForm-label"] ${PREVIEW_TOKEN_NAME} - user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] + user clicks checkbox by selector css:input[id="apiDataSetTokenCreateForm-agreeTerms"] user clicks button Continue user waits until page finishes loading @@ -238,9 +264,9 @@ User verifies the relevant fields on the active preview token page user waits until h2 is visible ${SUBJECT_NAME_1} user checks page contains Reference: ${PREVIEW_TOKEN_NAME} - ${current_time_tomorrow}= get current local datetime %-I:%M %p 1 + ${current_time_tomorrow}= get local browser date and time offset_days=1 format_string=%-I:%M %p user checks page contains - ... The token expires: tomorrow at ${time_end} (local time) + ... The token expires: tomorrow at ${current_time_tomorrow} (local time) user checks page contains button Copy preview token user checks page contains button Revoke preview token @@ -292,55 +318,72 @@ User verifies the row headings and contents in 'Data set details' section user checks row headings within the api data set section Publication user checks row headings within the api data set section Release user checks row headings within the api data set section Release type - user checks row headings within the api data set section Geographic levels - user checks row headings within the api data set section Indicators + user checks row headings within the api data set section Geographic levels + user checks row headings within the api data set section Indicators user checks row headings within the api data set section Filters user checks row headings within the api data set section Time period - user checks row headings within the api data set section Notifications - - user checks contents inside the cell value Test theme css: #dataSetDetails [data-testid="Theme-value"] - user checks contents inside the cell value ${PUBLICATION_NAME} css:#dataSetDetails [data-testid="Publication-value"] - user checks contents inside the cell value Academic year Q1 3000/01 css:#dataSetDetails [data-testid="Release-value"] - User checks contents inside the release type Accredited official statistics css:#dataSetDetails [data-testid="Release type-value"] > button - user checks contents inside the cell value National css:#dataSetDetails [data-testid="Geographic levels-value"] - - user checks contents inside the cell value Lower quartile annualised earnings css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(1) - user checks contents inside the cell value Median annualised earnings css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(2) - user checks contents inside the cell value Number of learners with earnings css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(3) - - user clicks button Show 1 more indicator css:#dataSetDetails [data-testid="Indicators-value"] - - user checks contents inside the cell value Upper quartile annualised earnings css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(4) - - user checks contents inside the cell value Cheese css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(1) - user checks contents inside the cell value Colour css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(2) - user checks contents inside the cell value Ethnicity group css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(3) - - user clicks button Show 4 more filters css:#dataSetDetails [data-testid="Filters-value"] - - user checks contents inside the cell value Gender css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(4) - user checks contents inside the cell value Level of learning css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(5) - user checks contents inside the cell value Number of years after achievement of learning aim css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(6) - user checks contents inside the cell value Provision css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(7) - - user checks contents inside the cell value 2012/13 css:#dataSetDetails [data-testid="Time period-value"] - user checks contents inside the cell value Get email updates about this API data set css:#dataSetDetails [data-testid="Notifications-value"] > a + user checks row headings within the api data set section Notifications + + user checks contents inside the cell value Test theme css: #dataSetDetails [data-testid="Theme-value"] + user checks contents inside the cell value ${PUBLICATION_NAME} + ... css:#dataSetDetails [data-testid="Publication-value"] + user checks contents inside the cell value Academic year Q1 3000/01 + ... css:#dataSetDetails [data-testid="Release-value"] + User checks contents inside the release type Accredited official statistics + ... css:#dataSetDetails [data-testid="Release type-value"] > button + user checks contents inside the cell value National + ... css:#dataSetDetails [data-testid="Geographic levels-value"] + + user checks contents inside the cell value Lower quartile annualised earnings + ... css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(1) + user checks contents inside the cell value Median annualised earnings + ... css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(2) + user checks contents inside the cell value Number of learners with earnings + ... css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(3) + + user clicks button Show 1 more indicator css:#dataSetDetails [data-testid="Indicators-value"] + + user checks contents inside the cell value Upper quartile annualised earnings + ... css:#dataSetDetails [data-testid="Indicators-value"] > ul > :nth-of-type(4) + + user checks contents inside the cell value Cheese + ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(1) + user checks contents inside the cell value Colour + ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(2) + user checks contents inside the cell value Ethnicity group + ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(3) + + user clicks button Show 4 more filters css:#dataSetDetails [data-testid="Filters-value"] + + user checks contents inside the cell value Gender + ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(4) + user checks contents inside the cell value Level of learning + ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(5) + user checks contents inside the cell value Number of years after achievement of learning aim + ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(6) + user checks contents inside the cell value Provision + ... css:#dataSetDetails [data-testid="Filters-value"] > ul > :nth-of-type(7) + + user checks contents inside the cell value 2012/13 css:#dataSetDetails [data-testid="Time period-value"] + user checks contents inside the cell value Get email updates about this API data set + ... css:#dataSetDetails [data-testid="Notifications-value"] > a User verifies the headings and contents in 'API version history' section user checks table column heading contains 1 1 Version css:section[id="apiVersionHistory"] user checks table column heading contains 1 2 Release css:section[id="apiVersionHistory"] - user checks table column heading contains 1 3 Status css:section[id="apiVersionHistory"] + user checks table column heading contains 1 3 Status css:section[id="apiVersionHistory"] + + user checks table cell contains 1 1 1.0 (current) xpath://section[@id="apiVersionHistory"] + user checks table cell contains 1 2 Academic year Q1 3000/01 xpath://section[@id="apiVersionHistory"] + user checks table cell contains 1 3 Published xpath://section[@id="apiVersionHistory"] - user checks table cell contains 1 1 1.0 (current) xpath://section[@id="apiVersionHistory"] - user checks table cell contains 1 2 Academic year Q1 3000/01 xpath://section[@id="apiVersionHistory"] - user checks table cell contains 1 3 Published xpath://section[@id="apiVersionHistory"] *** Keywords *** User checks contents inside the release type - [Arguments] ${expected_text} ${locator} - ${full_text}= get text ${locator} + [Arguments] ${expected_text} ${locator} + ${full_text}= get text ${locator} # Split and remove the part after '?' and strip whitespace ${button_text}= set variable ${full_text.split('?')[0].strip()} diff --git a/tests/robot-tests/tests/public_api/public_api_restricted.robot b/tests/robot-tests/tests/public_api/public_api_restricted.robot index 7d50cdeb648..998e3e19367 100644 --- a/tests/robot-tests/tests/public_api/public_api_restricted.robot +++ b/tests/robot-tests/tests/public_api/public_api_restricted.robot @@ -1,4 +1,3 @@ - *** Settings *** Library ../libs/admin_api.py Resource ../libs/admin-common.robot @@ -12,16 +11,14 @@ Suite Teardown user closes the browser Test Setup fail test fast if required - *** Variables *** -${PUBLICATION_NAME}= UI tests - Public API - restricted %{RUN_IDENTIFIER} -${RELEASE_NAME}= Financial year 3000-01 -${SUBJECT_NAME_1}= UI test subject 1 -${SUBJECT_NAME_2}= UI test subject 2 -${SUBJECT_NAME_3}= UI test subject 3 -${SUBJECT_NAME_4}= UI test subject 4 -${SUBJECT_NAME_5}= UI test subject 5 - +${PUBLICATION_NAME} UI tests - Public API - restricted %{RUN_IDENTIFIER} +${RELEASE_NAME} Financial year 3000-01 +${SUBJECT_NAME_1} UI test subject 1 +${SUBJECT_NAME_2} UI test subject 2 +${SUBJECT_NAME_3} UI test subject 3 +${SUBJECT_NAME_4} UI test subject 4 +${SUBJECT_NAME_5} UI test subject 5 *** Test Cases *** @@ -36,8 +33,10 @@ Verify release summary user verifies release summary Financial year 3000-01 Accredited official statistics Upload data files - user uploads subject and waits until complete ${SUBJECT_NAME_1} seven_filters.csv seven_filters.meta.csv ${PUBLIC_API_FILES_DIR} - user uploads subject and waits until complete ${SUBJECT_NAME_2} tiny-two-filters.csv tiny-two-filters.meta.csv ${PUBLIC_API_FILES_DIR} + user uploads subject and waits until complete ${SUBJECT_NAME_1} seven_filters.csv seven_filters.meta.csv + ... ${PUBLIC_API_FILES_DIR} + user uploads subject and waits until complete ${SUBJECT_NAME_2} tiny-two-filters.csv + ... tiny-two-filters.meta.csv ${PUBLIC_API_FILES_DIR} Add data guidance to subjects user clicks link Data and files @@ -65,7 +64,7 @@ Create 1st API dataset user clicks button Create API data set ${modal}= user waits until modal is visible Create a new API data set - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_1} + user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_1} user clicks button Confirm new API data set user waits until page finishes loading @@ -79,7 +78,7 @@ Create 2nd API dataset user clicks link Back to API data sets user clicks button Create API data set ${modal}= user waits until modal is visible Create a new API data set - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_2} + user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_2} user clicks button Confirm new API data set user waits until page finishes loading @@ -93,12 +92,11 @@ Verify the contents inside the 'Draft API datasets' table user clicks link Back to API data sets user waits until h3 is visible Draft API data sets - - user checks table column heading contains 1 1 Draft version xpath://table[@data-testid='draft-api-data-sets'] - user checks table column heading contains 1 2 Name xpath://table[@data-testid='draft-api-data-sets'] - user checks table column heading contains 1 3 Status xpath://table[@data-testid='draft-api-data-sets'] - user checks table column heading contains 1 4 Actions xpath://table[@data-testid='draft-api-data-sets'] - + user checks table column heading contains 1 1 Draft version + ... xpath://table[@data-testid='draft-api-data-sets'] + user checks table column heading contains 1 2 Name xpath://table[@data-testid='draft-api-data-sets'] + user checks table column heading contains 1 3 Status xpath://table[@data-testid='draft-api-data-sets'] + user checks table column heading contains 1 4 Actions xpath://table[@data-testid='draft-api-data-sets'] user checks table cell contains 1 1 v1.0 xpath://table[@data-testid='draft-api-data-sets'] user checks table cell contains 1 3 Ready xpath://table[@data-testid='draft-api-data-sets'] @@ -112,36 +110,56 @@ Click on 'View Details' link user checks table headings for Draft version details table User checks row data contents inside the 'Draft API datasets' summary table - user checks contents inside the cell value v1.0 xpath://dl[@data-testid="draft-version-summary"]/div/dd[@data-testid='Version-value']/strong - user checks contents inside the cell value Ready xpath:(//div[@data-testid="Status"]//dd[@data-testid="Status-value"]//strong)[2] - user checks contents inside the cell value Financial year 3000-01 xpath:(//div[@data-testid="Release"]//dd[@data-testid="Release-value"]//a)[1] - user checks contents inside the cell value ${SUBJECT_NAME_1} xpath://div[@data-testid="Data set file"]//dd[@data-testid="Data set file-value"] - user checks contents inside the cell value National xpath://div[@data-testid="Geographic levels"]//dd[@data-testid="Geographic levels-value"] - user checks contents inside the cell value 2012/13 xpath://div[@data-testid="Time periods"]//dd[@data-testid="Time periods-value"] - - - user checks contents inside the cell value Lower quartile annualised earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[1] - user checks contents inside the cell value Median annualised earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[2] - user checks contents inside the cell value Number of learners with earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[3] - - user clicks button Show 1 more indicator xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"] - - user checks contents inside the cell value Upper quartile annualised earnings xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[4] - - user checks contents inside the cell value Cheese xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[1] - user checks contents inside the cell value Colour xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[2] - user checks contents inside the cell value Ethnicity group xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[3] - - user clicks button Show 4 more filters xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"] - - user checks contents inside the cell value Gender xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[4] - user checks contents inside the cell value Level of learning xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[5] - user checks contents inside the cell value Number of years after achievement of learning aim xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[6] - user checks contents inside the cell value Provision xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[7] - - user checks contents inside the cell value Preview API data set xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[1]/a - user checks contents inside the cell value View preview token log xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[2]/a - user checks contents inside the cell value Remove draft version xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[3]/button + user checks contents inside the cell value v1.0 + ... xpath://dl[@data-testid="draft-version-summary"]/div/dd[@data-testid='Version-value']/strong + user checks contents inside the cell value Ready + ... xpath:(//div[@data-testid="Status"]//dd[@data-testid="Status-value"]//strong)[2] + user checks contents inside the cell value Financial year 3000-01 + ... xpath:(//div[@data-testid="Release"]//dd[@data-testid="Release-value"]//a)[1] + user checks contents inside the cell value ${SUBJECT_NAME_1} + ... xpath://div[@data-testid="Data set file"]//dd[@data-testid="Data set file-value"] + user checks contents inside the cell value National + ... xpath://div[@data-testid="Geographic levels"]//dd[@data-testid="Geographic levels-value"] + user checks contents inside the cell value 2012/13 + ... xpath://div[@data-testid="Time periods"]//dd[@data-testid="Time periods-value"] + + user checks contents inside the cell value Lower quartile annualised earnings + ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[1] + user checks contents inside the cell value Median annualised earnings + ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[2] + user checks contents inside the cell value Number of learners with earnings + ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[3] + + user clicks button Show 1 more indicator + ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"] + + user checks contents inside the cell value Upper quartile annualised earnings + ... xpath://div[@data-testid="Indicators"]//dd[@data-testid="Indicators-value"]/ul/li[4] + + user checks contents inside the cell value Cheese + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[1] + user checks contents inside the cell value Colour + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[2] + user checks contents inside the cell value Ethnicity group + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[3] + + user clicks button Show 4 more filters xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"] + + user checks contents inside the cell value Gender + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[4] + user checks contents inside the cell value Level of learning + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[5] + user checks contents inside the cell value Number of years after achievement of learning aim + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[6] + user checks contents inside the cell value Provision + ... xpath://div[@data-testid="Filters"]//dd[@data-testid="Filters-value"]/ul/li[7] + + user checks contents inside the cell value Preview API data set + ... xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[1]/a + user checks contents inside the cell value View preview token log + ... xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[2]/a + user checks contents inside the cell value Remove draft version + ... xpath://div[@data-testid="Actions"]//dd[@data-testid="Actions-value"]/ul/li[3]/button Add headline text block to Content page user navigates to content page ${PUBLICATION_NAME} @@ -175,7 +193,7 @@ Create a new API dataset version through the first amendment using the invalid s user clicks button Create API data set ${modal}= user waits until modal is visible Create a new API data set - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_3} + user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_3} user clicks button Confirm new API data set user waits until page finishes loading @@ -189,36 +207,34 @@ Verify the contents inside the 'Draft API datasets' table after the invalid impo user clicks link Back to API data sets user waits until h3 is visible Draft API data sets - user checks table column heading contains 1 1 Draft version xpath://table[@data-testid='draft-api-data-sets'] - user checks table column heading contains 1 2 Name xpath://table[@data-testid='draft-api-data-sets'] - user checks table column heading contains 1 3 Status xpath://table[@data-testid='draft-api-data-sets'] - user checks table column heading contains 1 4 Actions xpath://table[@data-testid='draft-api-data-sets'] - + user checks table column heading contains 1 1 Draft version + ... xpath://table[@data-testid='draft-api-data-sets'] + user checks table column heading contains 1 2 Name xpath://table[@data-testid='draft-api-data-sets'] + user checks table column heading contains 1 3 Status xpath://table[@data-testid='draft-api-data-sets'] + user checks table column heading contains 1 4 Actions xpath://table[@data-testid='draft-api-data-sets'] user checks table cell contains 1 1 v1.0 xpath://table[@data-testid='draft-api-data-sets'] user checks table cell contains 1 3 Failed xpath://table[@data-testid='draft-api-data-sets'] Verify the contents inside the 'Live API datasets' table after the invalid import fails - user checks table column heading contains 1 1 Version xpath://table[@data-testid='live-api-data-sets'] - user checks table column heading contains 1 2 Name xpath://table[@data-testid='live-api-data-sets'] - user checks table column heading contains 1 3 Actions xpath://table[@data-testid='live-api-data-sets'] - + user checks table column heading contains 1 1 Version xpath://table[@data-testid='live-api-data-sets'] + user checks table column heading contains 1 2 Name xpath://table[@data-testid='live-api-data-sets'] + user checks table column heading contains 1 3 Actions xpath://table[@data-testid='live-api-data-sets'] - user checks table cell contains 1 1 v1.0 xpath://table[@data-testid='live-api-data-sets'] - user checks table cell contains 1 2 ${SUBJECT_NAME_1} xpath://table[@data-testid='live-api-data-sets'] + user checks table cell contains 1 1 v1.0 xpath://table[@data-testid='live-api-data-sets'] + user checks table cell contains 1 2 ${SUBJECT_NAME_1} xpath://table[@data-testid='live-api-data-sets'] - user checks table cell contains 2 1 v1.0 xpath://table[@data-testid='live-api-data-sets'] - user checks table cell contains 2 2 ${SUBJECT_NAME_2} xpath://table[@data-testid='live-api-data-sets'] + user checks table cell contains 2 1 v1.0 xpath://table[@data-testid='live-api-data-sets'] + user checks table cell contains 2 2 ${SUBJECT_NAME_2} xpath://table[@data-testid='live-api-data-sets'] Add release note for the first release amendment user clicks link Content user adds a release note Test release note two - ${date} get current datetime %-d %B %Y + ${date}= get london date user waits until element contains css:#release-notes li:nth-of-type(1) time ${date} user waits until element contains css:#release-notes li:nth-of-type(1) p Test release note two - # When processing large API datasets, the current EES system returns one of two errors depending on the processing speed. # Additionally, there's an active bug ticket (EES-5420) - large data files are failing to create API datasets. # In response, I have added checks to handle either outcome. @@ -236,7 +252,8 @@ Create a second draft release via api user creates release from publication page ${PUBLICATION_NAME} Academic year 3010 Upload subject to second release - user uploads subject and waits until complete ${SUBJECT_NAME_4} seven_filters_minor_update.csv seven_filters_minor_update.meta.csv ${PUBLIC_API_FILES_DIR} + user uploads subject and waits until complete ${SUBJECT_NAME_4} seven_filters_minor_update.csv + ... seven_filters_minor_update.meta.csv ${PUBLIC_API_FILES_DIR} Add data guidance to second release user clicks link Data and files @@ -262,10 +279,11 @@ Create a different version of an API dataset (minor version) user waits until h3 is visible Current live API data sets user checks table column heading contains 1 1 Version xpath://table[@data-testid="live-api-data-sets"] - user clicks button in table cell 1 3 Create new version xpath://table[@data-testid="live-api-data-sets"] + user clicks button in table cell 1 3 Create new version + ... xpath://table[@data-testid="live-api-data-sets"] ${modal}= user waits until modal is visible Create a new API data set version - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_4} + user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_4} user clicks button Confirm new data set version user waits until page finishes loading @@ -273,8 +291,10 @@ Create a different version of an API dataset (minor version) Validate the summary contents inside the 'draft version details' table user waits until h3 is visible Draft version details - user waits until element contains css:dl[data-testid="draft-version-summary"] > div:nth-of-type(1) > dt + dd v1.1 %{WAIT_LONG} - user waits until element contains css=dl[data-testid="draft-version-summary"] > div:nth-of-type(2) > dt + dd Action required %{WAIT_LONG} + user waits until element contains css:dl[data-testid="draft-version-summary"] > div:nth-of-type(1) > dt + dd + ... v1.1 %{WAIT_LONG} + user waits until element contains css=dl[data-testid="draft-version-summary"] > div:nth-of-type(2) > dt + dd + ... Action required %{WAIT_LONG} ${mapping_status}= get text css:dl[data-testid="draft-version-summary"] > div:nth-of-type(2) > dt + dd should be equal as strings ${mapping_status} Action required @@ -295,7 +315,8 @@ Create a third draft release via api user creates release from publication page ${PUBLICATION_NAME} Academic year 3020 Upload subject to the third release - user uploads subject and waits until complete ${SUBJECT_NAME_5} institution_and_provider.csv institution_and_provider.meta.csv ${PUBLIC_API_FILES_DIR} + user uploads subject and waits until complete ${SUBJECT_NAME_5} institution_and_provider.csv + ... institution_and_provider.meta.csv ${PUBLIC_API_FILES_DIR} Add data guidance to the third release user clicks link Data and files @@ -321,21 +342,23 @@ Create a different version of API dataset (major version) for the third release user waits until h3 is visible Current live API data sets user checks table column heading contains 1 1 Version xpath://table[@data-testid="live-api-data-sets"] - user clicks button in table cell 1 3 Create new version xpath://table[@data-testid="live-api-data-sets"] + user clicks button in table cell 1 3 Create new version + ... xpath://table[@data-testid="live-api-data-sets"] ${modal}= user waits until modal is visible Create a new API data set version - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_5} + user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_5} user clicks button Confirm new data set version - user waits until page finishes loading user waits until modal is not visible Create a new API data set version %{WAIT_LONG} Validate the summary contents inside the 'draft version details' table for the third release user waits until h3 is visible Draft version details - user waits until element contains css:dl[data-testid="draft-version-summary"] > div:nth-of-type(1) > dt + dd v2.0 %{WAIT_LONG} - user waits until element contains css:dl[data-testid="draft-version-summary"] > div:nth-of-type(2) > dt + dd Action required %{WAIT_LONG} + user waits until element contains css:dl[data-testid="draft-version-summary"] > div:nth-of-type(1) > dt + dd + ... v2.0 %{WAIT_LONG} + user waits until element contains css:dl[data-testid="draft-version-summary"] > div:nth-of-type(2) > dt + dd + ... Action required %{WAIT_LONG} ${mapping_status}= get text css:dl[data-testid="draft-version-summary"] > div:nth-of-type(2) > dt + dd should be equal as strings ${mapping_status} Action required diff --git a/tests/robot-tests/tests/seed_data/seed_data_common.robot b/tests/robot-tests/tests/seed_data/seed_data_common.robot index 2bc0a6af735..9af8e3b0dcd 100644 --- a/tests/robot-tests/tests/seed_data/seed_data_common.robot +++ b/tests/robot-tests/tests/seed_data/seed_data_common.robot @@ -46,9 +46,9 @@ user creates a fully populated approved release ... ${RELEASE_YEAR} ... ${RELEASE_TYPE} ${days_until_release}= set variable 10000 - ${publish_date_day}= get current datetime %-d ${days_until_release} - ${publish_date_month}= get current datetime %-m ${days_until_release} - ${publish_date_year}= get current datetime %Y ${days_until_release} + ${publish_date_day}= get london day of month offset_days=${days_until_release} + ${publish_date_month}= get london month date offset_days=${days_until_release} + ${publish_date_year}= get london year offset_days=${days_until_release} user approves release for scheduled publication ... ${publish_date_day} ... ${publish_date_month} From 3c734abdb0903b0fa7e1bb392994ea27e0cd99ee Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 8 Oct 2024 13:30:16 +0100 Subject: [PATCH 38/80] EES-5526 - removed training forward slash from suite name in UI test pipeline to prevent test failure ZIP filename being invalid --- azure-pipelines-ui-tests.dfe.yml | 2 +- tests/robot-tests/tests/libs/dates_and_times.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/azure-pipelines-ui-tests.dfe.yml b/azure-pipelines-ui-tests.dfe.yml index 892f400448a..65a1176703c 100644 --- a/azure-pipelines-ui-tests.dfe.yml +++ b/azure-pipelines-ui-tests.dfe.yml @@ -49,7 +49,7 @@ jobs: displayName: Public UI tests inputs: scriptPath: tests/robot-tests/scripts/run_tests_pipeline.py - arguments: --admin-pass "test" --analyst-pass "test" --expiredinvite-pass "test" --noinvite-pass "test" --pendinginvite-pass "test" --env "dev" --file "tests/general_public/" --processes 4 --rerun-attempts 3 + arguments: --admin-pass "test" --analyst-pass "test" --expiredinvite-pass "test" --noinvite-pass "test" --pendinginvite-pass "test" --env "dev" --file "tests/general_public" --processes 4 --rerun-attempts 3 workingDirectory: tests/robot-tests env: SLACK_APP_TOKEN: $(ees-alerts-slackapptoken) diff --git a/tests/robot-tests/tests/libs/dates_and_times.py b/tests/robot-tests/tests/libs/dates_and_times.py index 22ff7ef7128..e9626b76118 100644 --- a/tests/robot-tests/tests/libs/dates_and_times.py +++ b/tests/robot-tests/tests/libs/dates_and_times.py @@ -1,6 +1,5 @@ import datetime import os - import pytz from tests.libs.selenium_elements import sl From f9465adcaefbf2f593df5db849ccf3d161890b55 Mon Sep 17 00:00:00 2001 From: dfe-sdt Date: Tue, 8 Oct 2024 15:43:36 +0000 Subject: [PATCH 39/80] chore(tests): update test snapshots --- tests/robot-tests/tests/snapshots/find_statistics_snapshot.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json b/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json index e243b63dad3..30a4d62c2be 100644 --- a/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json +++ b/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json @@ -525,7 +525,7 @@ "theme": "Pupils and schools" }, { - "publication_summary": "Pupil attendance and absence data including termly national statistics and fortnightly experimental statistics derived from DfE\u2019s regular attendance data", + "publication_summary": "Pupil attendance and absence data including termly national statistics and fortnightly statistics in development derived from DfE\u2019s regular attendance data", "publication_title": "Pupil attendance in schools", "published": "26 Sep 2024", "release_type": "Official statistics in development", From 09596c859ade2f6c5844e5454237fb3535747d08 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Tue, 8 Oct 2024 18:06:09 +0100 Subject: [PATCH 40/80] EES-5526 - responding to PR comments. Made date time code a bit clearer by removing some indirection --- tests/robot-tests/tests/libs/dates_and_times.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/robot-tests/tests/libs/dates_and_times.py b/tests/robot-tests/tests/libs/dates_and_times.py index e9626b76118..5c146c32693 100644 --- a/tests/robot-tests/tests/libs/dates_and_times.py +++ b/tests/robot-tests/tests/libs/dates_and_times.py @@ -1,5 +1,6 @@ import datetime import os + import pytz from tests.libs.selenium_elements import sl @@ -17,19 +18,19 @@ def get_local_browser_date_and_time(offset_days: int = 0, format_string: str = " def get_london_day_of_month(offset_days: int = 0) -> str: - return get_london_date_and_time(offset_days, "%-d") + return _get_date_and_time(offset_days, "%-d", "Europe/London") def get_london_month_date(offset_days: int = 0) -> str: - return get_london_date_and_time(offset_days, "%-m") + return _get_date_and_time(offset_days, "%-m", "Europe/London") def get_london_month_word(offset_days: int = 0) -> str: - return get_london_date_and_time(offset_days, "%B") + return _get_date_and_time(offset_days, "%B", "Europe/London") def get_london_year(offset_days: int = 0) -> str: - return get_london_date_and_time(offset_days, "%Y") + return _get_date_and_time(offset_days, "%Y", "Europe/London") def _get_browser_timezone(): From 19c0bb92d14df75402826b01caae2237a0289c19 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Wed, 9 Oct 2024 10:50:50 +0100 Subject: [PATCH 41/80] Adding guidance for setting "max_user_watches" limit on Linux machines in order to run the integration tests --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index fbdcfa3d1dd..6f01b00d937 100644 --- a/README.md +++ b/README.md @@ -700,6 +700,30 @@ cd src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api dotnet ef migrations add EES1234_MigrationNameHere --context PublicDataDbContext --project ../GovUk.Education.ExploreEducationStatistics.Public.Data.Model -v ``` +### Running backend tests + +The backend c# projects have a number of unit and integration tests. From the project root, run: + +```sh +cd src +dotnet clean +dotnet test +``` + +Note that the `clean` is necessary due to an issue with [AspectInjector](https://github.com/pamidur/aspect-injector) +whereby compilation of code over already-compiled code will add AOP execution code on top of existing AOP execution +code, leading to AOP code being invoked multiple times rather than just once. This would result in test failures, as we +assert that AOP code is executed only once. + +#### Configuring Linux for running unit and integration tests + +Due to the resource requirements of the integration tests, Linux users need to ensure that the system running the tests +is capable of doing so. The `max_user_watches` setting must be set to a high enough limit, for example by running: + +```sh +echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf +``` + ### Resetting Azurite During development you might want to reset your Azurite instance to clear out all data from From 69b37304ed0c96e4442198c807a70a476cd3c992 Mon Sep 17 00:00:00 2001 From: Tom Jones Date: Wed, 9 Oct 2024 11:36:08 +0100 Subject: [PATCH 42/80] EES-5502: Replace WYSIWYG link text validation to instead trigger based on a pre-defined list of words. --- .../editable/InvalidContentDetails.tsx | 58 ++++----------- .../__tests__/InvalidContentDetails.test.tsx | 24 ++----- .../__tests__/getInvalidContent.test.tsx | 45 +++--------- .../editable/utils/getInvalidContent.ts | 72 +++++++++++++++---- 4 files changed, 87 insertions(+), 112 deletions(-) diff --git a/src/explore-education-statistics-admin/src/components/editable/InvalidContentDetails.tsx b/src/explore-education-statistics-admin/src/components/editable/InvalidContentDetails.tsx index d2d8e3a7385..5b9b5bf6fc4 100644 --- a/src/explore-education-statistics-admin/src/components/editable/InvalidContentDetails.tsx +++ b/src/explore-education-statistics-admin/src/components/editable/InvalidContentDetails.tsx @@ -10,8 +10,8 @@ export default function InvalidContentDetails({ }: { errors: InvalidContentError[]; }) { - const clickHereLinkTextErrors = errors.filter( - error => error.type === 'clickHereLinkText', + const badLinkTextErrors = errors.filter( + error => error.type === 'badLinkText', ); const repeatedLinkTextErrors = errors.filter( error => error.type === 'repeatedLinkText', @@ -20,9 +20,6 @@ export default function InvalidContentDetails({ repeatedLinkTextErrors, 'message', ); - const oneWordLinkTextErrors = errors.filter( - error => error.type === 'oneWordLinkText', - ); const urlLinkTextErrors = errors.filter( error => error.type === 'urlLinkText', ); @@ -43,8 +40,11 @@ export default function InvalidContentDetails({ <>

The following accessibility problems have been found:

    - {!!clickHereLinkTextErrors.length && ( + {!!badLinkTextErrors.length && ( ( +
  • {error.message}
    • + ))} modalContent={ <>

    @@ -53,15 +53,17 @@ export default function InvalidContentDetails({ descriptive and understandable on their own.

    - Avoid using "click here" as link text as it does not describe - where the link is to. + Avoid using phrases such as "click here" as it does not + describe where the link is to. Similarly, you should try to + avoid short, or single word links because this limits how + descriptive and user friendly it can be, and can also create + problems for users with limited dexterity to click on a small + area.

    } - modalTitle='"Click here" links' - text={`${clickHereLinkTextErrors.length} "click here" ${ - clickHereLinkTextErrors.length === 1 ? 'link' : 'links' - }`} + modalTitle="Bad link text" + text={`${badLinkTextErrors.length} bad link text`} /> )} @@ -96,38 +98,6 @@ export default function InvalidContentDetails({ /> )} - {!!oneWordLinkTextErrors.length && ( - ( -
  • {error.message}
    • - ))} - modalContent={ - <> -

    - Links are often viewed out of context and used to help - navigate pages like headers, it is important that they are - descriptive and understandable on their own. -

    -

    Avoid one word links because:

    -
      -
    • - they can create problems for users with limited dexterity to - click on a small area -
      • -
    • - having a single word limits how descriptive and user - friendly it can be -
      • -
    - - } - modalTitle="One word link text" - text={`${oneWordLinkTextErrors.length} ${ - oneWordLinkTextErrors.length === 1 ? 'link' : 'links' - } with one word link text`} - /> - )} - {!!urlLinkTextErrors.length && ( ( diff --git a/src/explore-education-statistics-admin/src/components/editable/__tests__/InvalidContentDetails.test.tsx b/src/explore-education-statistics-admin/src/components/editable/__tests__/InvalidContentDetails.test.tsx index a98af4b24d0..500349e74fe 100644 --- a/src/explore-education-statistics-admin/src/components/editable/__tests__/InvalidContentDetails.test.tsx +++ b/src/explore-education-statistics-admin/src/components/editable/__tests__/InvalidContentDetails.test.tsx @@ -3,20 +3,6 @@ import { InvalidContentError } from '@admin/components/editable/utils/getInvalid import { render, screen } from '@testing-library/react'; describe('InvalidContentDetails', () => { - test('renders clickHereLink errors', () => { - const testErrors: InvalidContentError[] = [ - { - type: 'clickHereLinkText', - }, - { - type: 'clickHereLinkText', - }, - ]; - render(); - - expect(screen.getByRole('button', { name: /2 "click here" links/ })); - }); - test('renders repeatedLinkText errors', () => { const testErrors: InvalidContentError[] = [ { @@ -40,21 +26,21 @@ describe('InvalidContentDetails', () => { expect(screen.getByText('Repeated link text: url-1, url-2')); }); - test('renders oneWordLinkText errors', () => { + test('renders badLinkText errors', () => { const testErrors: InvalidContentError[] = [ { - type: 'oneWordLinkText', - message: 'word', + type: 'badLinkText', + message: 'learn more', }, ]; render(); expect( screen.getByRole('button', { - name: /1 link with one word link text/, + name: /1 bad link text/, }), ); - expect(screen.getByText('word')); + expect(screen.getByText('learn more')); }); test('renders urlLinkText errors', () => { diff --git a/src/explore-education-statistics-admin/src/components/editable/utils/__tests__/getInvalidContent.test.tsx b/src/explore-education-statistics-admin/src/components/editable/utils/__tests__/getInvalidContent.test.tsx index 59b0c48e62a..1bb59f638c0 100644 --- a/src/explore-education-statistics-admin/src/components/editable/utils/__tests__/getInvalidContent.test.tsx +++ b/src/explore-education-statistics-admin/src/components/editable/utils/__tests__/getInvalidContent.test.tsx @@ -335,7 +335,7 @@ describe('getInvalidContent', () => { ]); }); - test('returns an error when the link text is just one word', () => { + test('returns an error when the link text is an inaccessible word or phrase', () => { const testContent: JsonElement[] = [ { name: 'paragraph', @@ -366,39 +366,7 @@ describe('getInvalidContent', () => { linkHref: 'https://bbc.co.uk', linkOpenInNewTab: true, }, - data: 'link', - }, - { - data: ' words', - }, - ], - }, - ]; - - const result = getInvalidContent(testContent); - - expect(result).toEqual([ - { - type: 'oneWordLinkText', - message: 'link', - }, - ]); - }); - - test('returns an error when the link text is "click here"', () => { - const testContent: JsonElement[] = [ - { - name: 'paragraph', - children: [ - { - data: 'words ', - }, - { - attributes: { - linkHref: 'https://gov.uk', - linkOpenInNewTab: true, - }, - data: 'link to something', + data: 'learn more', }, { data: ' words', @@ -416,7 +384,7 @@ describe('getInvalidContent', () => { linkHref: 'https://bbc.co.uk', linkOpenInNewTab: true, }, - data: 'click here', + data: ' learn more ', }, { data: ' words', @@ -429,7 +397,12 @@ describe('getInvalidContent', () => { expect(result).toEqual([ { - type: 'clickHereLinkText', + type: 'badLinkText', + message: 'learn more', + }, + { + type: 'badLinkText', + message: ' learn more ', }, ]); }); diff --git a/src/explore-education-statistics-admin/src/components/editable/utils/getInvalidContent.ts b/src/explore-education-statistics-admin/src/components/editable/utils/getInvalidContent.ts index 6bd5211252f..5149bf364df 100644 --- a/src/explore-education-statistics-admin/src/components/editable/utils/getInvalidContent.ts +++ b/src/explore-education-statistics-admin/src/components/editable/utils/getInvalidContent.ts @@ -3,9 +3,8 @@ import parseNumber from '@common/utils/number/parseNumber'; export interface InvalidContentError { type: - | 'clickHereLinkText' | 'repeatedLinkText' - | 'oneWordLinkText' + | 'badLinkText' | 'urlLinkText' | 'skippedHeadingLevel' | 'missingTableHeaders' @@ -95,13 +94,6 @@ export default function getInvalidContent( }, []); allLinks.forEach(link => { - if (link.data?.toLowerCase().trim() === 'click here') { - errors.push({ - type: 'clickHereLinkText', - }); - return; - } - if (link.data === link.attributes?.linkHref) { errors.push({ type: 'urlLinkText', @@ -110,13 +102,67 @@ export default function getInvalidContent( return; } - // exclude glossary links + const badLinkTextWords = [ + 'click', + 'csv', + 'continue', + 'dashboard', + 'document', + 'download', + 'file', + 'form', + 'guidance', + 'here', + 'info', + 'information', + 'jpeg', + 'jpg', + 'learn', + 'link', + 'more', + 'next', + 'page', + 'pdf', + 'previous', + 'read', + 'site', + 'svg', + 'this', + 'web', + 'webpage', + 'website', + 'word', + 'xslx', + 'click here', + 'click this link', + 'download csv', + 'download document', + 'download file', + 'download here', + 'download jpg', + 'download jpeg', + 'download pdf', + 'download png', + 'download svg', + 'download word', + 'download xslx', + 'further information', + 'go here', + 'learn more', + 'link to', + 'read more', + 'this page', + 'visit this', + 'web page', + 'web site', + ]; + if ( - link.data?.split(' ').length === 1 && - !link.attributes?.linkHref?.includes('/glossary') + link.data && + badLinkTextWords.includes(link.data.trim().toLowerCase()) ) { errors.push({ - type: 'oneWordLinkText', + type: 'badLinkText', message: link.data, }); return; From 843cce7b85851d51b3e4b4fc0b18e86f92543c33 Mon Sep 17 00:00:00 2001 From: "rian.thwaite" Date: Wed, 9 Oct 2024 14:07:16 +0100 Subject: [PATCH 43/80] EES-5527 fix formatting --- .../content/components/EditableKeyStatDataBlock.tsx | 8 +++++--- .../release/content/components/EditableKeyStatText.tsx | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatDataBlock.tsx b/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatDataBlock.tsx index 7d14c244047..fe2dc56baf8 100644 --- a/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatDataBlock.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatDataBlock.tsx @@ -33,7 +33,7 @@ export default function EditableKeyStatDataBlock({ onRemove, onSubmit, }: EditableKeyStatDataBlockProps) { - const [keyStatisticId, setKeyStatisticId] = useState(""); + const [keyStatisticId, setKeyStatisticId] = useState(''); const [showForm, toggleShowForm] = useToggle(false); const { @@ -47,7 +47,7 @@ export default function EditableKeyStatDataBlock({ const handleSubmit = useCallback( async (values: KeyStatDataBlockFormValues) => { await onSubmit(values); - setKeyStatisticId(""); + setKeyStatisticId(''); toggleShowForm.off(); }, [onSubmit, toggleShowForm, setKeyStatisticId], @@ -78,7 +78,9 @@ export default function EditableKeyStatDataBlock({ return ( keyStatTitle === keyStatisticId)} + keyStatisticGuidanceTitles={keyStatisticGuidanceTitles?.filter( + keyStatTitle => keyStatTitle === keyStatisticId, + )} title={title} statistic={statistic} isReordering={isReordering} diff --git a/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatText.tsx b/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatText.tsx index 8c963baad8b..b4aadedddc5 100644 --- a/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatText.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatText.tsx @@ -25,13 +25,13 @@ export default function EditableKeyStatText({ onRemove, onSubmit, }: EditableKeyStatTextProps) { - const [keyStatisticId, setKeyStatisticId] = useState(""); + const [keyStatisticId, setKeyStatisticId] = useState(''); const [showForm, toggleShowForm] = useToggle(false); const handleSubmit = useCallback( async (values: KeyStatTextFormValues) => { await onSubmit(values); - setKeyStatisticId(""); + setKeyStatisticId(''); toggleShowForm.off(); }, [onSubmit, setKeyStatisticId, toggleShowForm], @@ -41,7 +41,9 @@ export default function EditableKeyStatText({ return ( keyStatTitle === keyStatisticId)} + keyStatisticGuidanceTitles={keyStatisticGuidanceTitles?.filter( + keyStatTitle => keyStatTitle === keyStatisticId, + )} isReordering={isReordering} testId={testId} onSubmit={handleSubmit} From ceb5601a33b60afb74526005e1e2dadaf87f2c17 Mon Sep 17 00:00:00 2001 From: Amy Benson Date: Thu, 10 Oct 2024 12:25:13 +0100 Subject: [PATCH 44/80] EES-5307 fix table tool publication error link --- .../src/modules/table-tool/components/PublicationForm.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/explore-education-statistics-common/src/modules/table-tool/components/PublicationForm.tsx b/src/explore-education-statistics-common/src/modules/table-tool/components/PublicationForm.tsx index d751ece4788..319c1260cd0 100644 --- a/src/explore-education-statistics-common/src/modules/table-tool/components/PublicationForm.tsx +++ b/src/explore-education-statistics-common/src/modules/table-tool/components/PublicationForm.tsx @@ -192,7 +192,6 @@ const PublicationForm = ({
    - id="publications" legend={ <> Select a publication From 89535cb44875f657667fa3a2c25145906e76a196 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Wed, 9 Oct 2024 18:03:49 +0100 Subject: [PATCH 45/80] EES-5559 Add `CloneExtensions` for shallow and deep copying --- .../Extensions/CloneExtensionsTests.cs | 113 ++++++++++++++++++ .../Extensions/CloneExtensions.cs | 15 +++ ...n.ExploreEducationStatistics.Common.csproj | 1 + 3 files changed, 129 insertions(+) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/CloneExtensionsTests.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/CloneExtensions.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/CloneExtensionsTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/CloneExtensionsTests.cs new file mode 100644 index 00000000000..5f2cdd7dfc5 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Common.Tests/Extensions/CloneExtensionsTests.cs @@ -0,0 +1,113 @@ +#nullable enable +using System.Collections.Generic; +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; +using Xunit; + +namespace GovUk.Education.ExploreEducationStatistics.Common.Tests.Extensions; + +public static class CloneExtensionsTests +{ + public class ShallowCloneTests + { + [Theory] + [InlineData(123)] + [InlineData(1.1)] + [InlineData(true)] + [InlineData(false)] + public void NonStringPrimitives(object value) + { + Assert.Equal(value, value.ShallowClone()); + } + + [Fact] + public void String() + { + const string value = "test"; + + // Need to be able to specify the type is + // a string to shallow clone it correctly. + Assert.Equal(value, value.ShallowClone()); + } + + [Fact] + public void Object() + { + var obj = new TestPerson + { + Name = "Test person", + Age = 123, + Addresses = + [ + new TestAddress { Line1 = "123 High Street" }, + new TestAddress { Line1 = "456 Low Street" }, + ] + }; + + var clone = obj.ShallowClone(); + + Assert.Equal(obj.Name, clone.Name); + Assert.Equal(obj.Age, clone.Age); + + Assert.Same(obj.Addresses, clone.Addresses); + Assert.Same(obj.Addresses[0], clone.Addresses[0]); + Assert.Same(obj.Addresses[1], clone.Addresses[1]); + } + } + + public class DeepCloneTests + { + [Theory] + [InlineData(123)] + [InlineData(1.1)] + [InlineData(true)] + [InlineData(false)] + [InlineData("test")] + public void Primitives(object value) + { + Assert.Equal(value, value.DeepClone()); + } + + [Fact] + public void Object() + { + var obj = new TestPerson + { + Name = "Test person", + Age = 123, + Addresses = + [ + new TestAddress { Line1 = "123 High Street" }, + new TestAddress { Line1 = "456 Low Street" }, + ] + }; + + var clone = obj.DeepClone(); + + Assert.Equal(obj.Name, clone.Name); + Assert.Equal(obj.Age, clone.Age); + + Assert.NotSame(obj.Addresses, clone.Addresses); + Assert.Equal(obj.Addresses.Count, clone.Addresses.Count); + + Assert.NotSame(obj.Addresses[0], clone.Addresses[0]); + Assert.Equal(obj.Addresses[0].Line1, clone.Addresses[0].Line1); + + Assert.NotSame(obj.Addresses[1], clone.Addresses[1]); + Assert.Equal(obj.Addresses[1].Line1, clone.Addresses[1].Line1); + } + } + + private class TestPerson + { + public required string Name { get; init; } + + public required int Age { get; init; } + + public List Addresses { get; init; } = []; + } + + private class TestAddress + { + public required string Line1 { get; init; } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/CloneExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/CloneExtensions.cs new file mode 100644 index 00000000000..d6edba9f242 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/Extensions/CloneExtensions.cs @@ -0,0 +1,15 @@ +#nullable enable +namespace GovUk.Education.ExploreEducationStatistics.Common.Extensions; + +public static class CloneExtensions +{ + public static T ShallowClone(this T obj) + { + return ObjectCloner.ObjectCloner.ShallowClone(obj); + } + + public static T DeepClone(this T obj) + { + return ObjectCloner.ObjectCloner.DeepClone(obj); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Common/GovUk.Education.ExploreEducationStatistics.Common.csproj b/src/GovUk.Education.ExploreEducationStatistics.Common/GovUk.Education.ExploreEducationStatistics.Common.csproj index 44c03d75a42..43e201c0d93 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Common/GovUk.Education.ExploreEducationStatistics.Common.csproj +++ b/src/GovUk.Education.ExploreEducationStatistics.Common/GovUk.Education.ExploreEducationStatistics.Common.csproj @@ -41,6 +41,7 @@ + From 64f3c8bac9d38c44c6b8a6805ec42113cdeb8117 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Thu, 10 Oct 2024 01:32:07 +0100 Subject: [PATCH 46/80] EES-5559 Fix filters and indicators not reusing public IDs This fixes filters and indicators previously being imported as new filters / indicators, even if they could be auto mapped to re-use any existing public IDs i.e. when their columns match with the previous version's filters / indicators. --- .../FilterMappingPlanGeneratorExtensions.cs | 49 +- .../DataSetVersionMapping.cs | 4 +- .../Functions/ImportMetadataFunctionTests.cs | 998 ++++++++++++++---- .../ProcessorTestData.cs | 81 ++ .../Repository/FilterMetaRepository.cs | 116 +- .../Repository/IndicatorMetaRepository.cs | 44 +- 6 files changed, 1004 insertions(+), 288 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/FilterMappingPlanGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/FilterMappingPlanGeneratorExtensions.cs index e7a8d7d5df1..43dddcd08e0 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/FilterMappingPlanGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/FilterMappingPlanGeneratorExtensions.cs @@ -1,3 +1,4 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Utils; @@ -13,35 +14,44 @@ public static Generator DefaultFilterMappingPlan(this DataFix public static Generator FilterMappingPlanFromFilterMeta( this DataFixture fixture, - List? sourceFilters = null, - List? targetFilters = null) + IEnumerable? sourceFilters = null, + IEnumerable? targetFilters = null) { var filterMappingPlanGenerator = fixture.Generator(); sourceFilters?.ForEach(sourceFilter => { - var filterMappingGenerator = fixture - .DefaultFilterMapping() - .WithSource(fixture - .DefaultMappableFilter() + var autoMappedFilter = targetFilters?.SingleOrDefault(f => f.Column == sourceFilter.Column); + + var filterMappingGenerator = fixture.DefaultFilterMapping() + .WithSource(fixture.DefaultMappableFilter() .WithLabel(sourceFilter.Label)) + .WithType(autoMappedFilter is not null ? MappingType.AutoMapped : MappingType.AutoNone) + .WithCandidateKey(autoMappedFilter?.Column) .WithPublicId(sourceFilter.PublicId); sourceFilter.Options.ForEach(option => { + var sourceKey = MappingKeyGenerators.FilterOptionMeta(option); + var sourceLink = sourceFilter.OptionLinks.SingleOrDefault(l => l.OptionId == option.Id); + + var autoMappedOption = autoMappedFilter?.Options + .SingleOrDefault(o => MappingKeyGenerators.FilterOptionMeta(o) == sourceKey); + filterMappingGenerator.AddOptionMapping( - sourceKey: MappingKeyGenerators.FilterOptionMeta(option), - fixture - .DefaultFilterOptionMapping() - .WithSource(fixture - .DefaultMappableFilterOption() + sourceKey: sourceKey, + mapping: fixture.DefaultFilterOptionMapping() + .WithSource(fixture.DefaultMappableFilterOption() .WithLabel(option.Label)) - .WithPublicId($"{sourceFilter.PublicId} :: {option.Label}")); + .WithType(autoMappedOption is not null ? MappingType.AutoMapped : MappingType.AutoNone) + .WithCandidateKey(autoMappedOption is not null + ? MappingKeyGenerators.FilterOptionMeta(autoMappedOption) + : null) + .WithPublicId(sourceLink?.PublicId ?? $"{sourceFilter.PublicId} :: {option.Label}") + ); }); - filterMappingPlanGenerator.AddFilterMapping( - columnName: sourceFilter.PublicId, - filterMappingGenerator); + filterMappingPlanGenerator.AddFilterMapping(sourceFilter.Column,filterMappingGenerator); }); targetFilters?.ForEach(targetFilter => @@ -54,14 +64,11 @@ public static Generator FilterMappingPlanFromFilterMeta( { filterCandidateGenerator.AddOptionCandidate( targetKey: MappingKeyGenerators.FilterOptionMeta(option), - fixture - .DefaultMappableFilterOption() + candidate: fixture.DefaultMappableFilterOption() .WithLabel(option.Label)); }); - filterMappingPlanGenerator.AddFilterCandidate( - columnName: targetFilter.PublicId, - filterCandidateGenerator); + filterMappingPlanGenerator.AddFilterCandidate(targetFilter.Column, filterCandidateGenerator); }); return filterMappingPlanGenerator; @@ -173,7 +180,7 @@ public static Generator WithManualNone( public static Generator WithCandidateKey( this Generator generator, - string candidateKey) + string? candidateKey) => generator.ForInstance(s => s.SetCandidateKey(candidateKey)); public static Generator AddOptionMapping( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionMapping.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionMapping.cs index a5e402b8ce6..dc50b957815 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionMapping.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionMapping.cs @@ -125,7 +125,7 @@ public enum MappingType ///
public abstract record MappableElement { - public required string Label { get; init; } + public required string Label { get; set; } } public abstract record MappableElementWithOptions @@ -146,7 +146,7 @@ public abstract record Mapping { public required TMappableElement Source { get; init; } - public required string PublicId { get; init; } + public required string PublicId { get; set; } [JsonConverter(typeof(JsonStringEnumConverter))] [Newtonsoft.Json.JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportMetadataFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportMetadataFunctionTests.cs index 8bcc8af2919..203b3927a8b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportMetadataFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ImportMetadataFunctionTests.cs @@ -25,26 +25,26 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests public abstract class ImportMetadataFunctionTests(ProcessorFunctionsIntegrationTestFixture fixture) : ProcessorFunctionsIntegrationTest(fixture) { - public class ImportMetadataTests(ProcessorFunctionsIntegrationTestFixture fixture) - : ImportMetadataFunctionTests(fixture) + private const DataSetVersionImportStage Stage = DataSetVersionImportStage.ImportingMetadata; + + public static readonly TheoryData TestDataFiles = new() { - private const DataSetVersionImportStage Stage = DataSetVersionImportStage.ImportingMetadata; + ProcessorTestData.AbsenceSchool, + }; - public static readonly TheoryData TestDataFiles = new() + public static readonly TheoryData TestDataFilesWithMetaInsertBatchSize = + new() { - ProcessorTestData.AbsenceSchool, + { ProcessorTestData.AbsenceSchool, 1 }, + { ProcessorTestData.AbsenceSchool, 1000 }, }; - public static readonly TheoryData TestDataFilesWithMetaInsertBatchSize = - new() - { - { ProcessorTestData.AbsenceSchool, 1 }, - { ProcessorTestData.AbsenceSchool, 1000 }, - }; - + public class ImportMetadataDbTests(ProcessorFunctionsIntegrationTestFixture fixture) + : ImportMetadataFunctionTests(fixture) + { [Theory] [MemberData(nameof(TestDataFiles))] - public async Task Success(ProcessorTestData testData) + public async Task InitialVersion_Success(ProcessorTestData testData) { var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); @@ -60,12 +60,12 @@ public async Task Success(ProcessorTestData testData) Assert.Equal(Stage, savedImport.Stage); Assert.Equal(DataSetVersionStatus.Processing, savedDataSetVersion.Status); - AssertDataSetVersionDirectoryContainsOnlyFiles(dataSetVersion, - [ + AssertDataSetVersionDirectoryContainsOnlyFiles( + dataSetVersion, DataSetFilenames.CsvDataFile, DataSetFilenames.CsvMetadataFile, DataSetFilenames.DuckDbDatabaseFile - ]); + ); Assert.Equal(testData.ExpectedTotalResults, savedDataSetVersion.TotalResults); @@ -97,7 +97,7 @@ public async Task Success(ProcessorTestData testData) [Theory] [MemberData(nameof(TestDataFiles))] - public async Task DataSetVersionMeta_CorrectGeographicLevels(ProcessorTestData testData) + public async Task InitialVersion_CorrectGeographicLevels(ProcessorTestData testData) { var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); @@ -117,7 +117,7 @@ public async Task DataSetVersionMeta_CorrectGeographicLevels(ProcessorTestData t [Theory] [MemberData(nameof(TestDataFilesWithMetaInsertBatchSize))] - public async Task DataSetVersionMeta_CorrectLocationOptions(ProcessorTestData testData, int metaInsertBatchSize) + public async Task InitialVersion_CorrectLocationOptions(ProcessorTestData testData, int metaInsertBatchSize) { var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); @@ -172,7 +172,7 @@ public async Task DataSetVersionMeta_CorrectLocationOptions(ProcessorTestData te [Theory] [MemberData(nameof(TestDataFilesWithMetaInsertBatchSize))] - public async Task DataSetVersionMeta_CorrectLocationOptions_WithMappings( + public async Task NextVersion_CorrectLocationOptions_WithMappings( ProcessorTestData testData, int metaInsertBatchSize) { @@ -290,7 +290,7 @@ await AddTestData(context => [Theory] [MemberData(nameof(TestDataFiles))] - public async Task DataSetVersionMeta_CorrectTimePeriods(ProcessorTestData testData) + public async Task InitialVersion_CorrectTimePeriods_NoExistingTimePeriods(ProcessorTestData testData) { var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); @@ -319,7 +319,9 @@ public async Task DataSetVersionMeta_CorrectTimePeriods(ProcessorTestData testDa [Theory] [MemberData(nameof(TestDataFilesWithMetaInsertBatchSize))] - public async Task DataSetVersionMeta_CorrectFilters(ProcessorTestData testData, int metaInsertBatchSize) + public async Task InitialVersion_CorrectFiltersAndOptions( + ProcessorTestData testData, + int metaInsertBatchSize) { var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); @@ -327,228 +329,762 @@ public async Task DataSetVersionMeta_CorrectFilters(ProcessorTestData testData, await using var publicDataDbContext = GetDbContext(); - var actualFilters = await publicDataDbContext.FilterMetas - .Include(fm => fm.Options.OrderBy(o => o.Label)) - .ThenInclude(fom => fom.MetaLinks) - .Where(fm => fm.DataSetVersionId == dataSetVersion.Id) - .OrderBy(fm => fm.Label) - .ToListAsync(); + var actualFilters = await GetDbFilterMetas(dataSetVersion.Id); - var globalOptionIndex = 0; + Assert.Equal(testData.ExpectedFilters.Count, actualFilters.Count); + Assert.All(testData.ExpectedFilters, (expectedFilter, filterIndex) => + { + var actualFilter = actualFilters[filterIndex]; + + AssertFiltersEqual(expectedFilter, actualFilter); + AssertAllFilterOptionsEqual(expectedFilter, actualFilter); + }); + } + + [Fact] + public async Task NextVersion_CorrectFilters_AutoMappedWithSamePublicIds() + { + var testData = ProcessorTestData.AbsenceSchool; + + var (sourceDataSetVersion, targetDataSetVersion, instanceId) = + await CreateDataSetInitialAndNextVersion( + nextVersionImportStage: DataSetVersionImportStage.ManualMapping, + nextVersionStatus: DataSetVersionStatus.Mapping); + + DataSetVersionMapping mapping = DataFixture.DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(sourceDataSetVersion.Id) + .WithTargetDataSetVersionId(targetDataSetVersion.Id) + .WithFilterMappingPlan( + DataFixture.FilterMappingPlanFromFilterMeta( + sourceFilters: testData.ExpectedFilters, + targetFilters: testData.ExpectedFilters + ) + ); + + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ImportMetadata(testData, targetDataSetVersion, instanceId); + + await using var publicDataDbContext = GetDbContext(); + + var actualFilters = await GetDbFilterMetas(targetDataSetVersion.Id); Assert.Equal(testData.ExpectedFilters.Count, actualFilters.Count); - Assert.All(testData.ExpectedFilters, - (expectedFilter, index) => - { - var actualFilter = actualFilters[index]; - actualFilter.AssertDeepEqualTo(expectedFilter, - notEqualProperties: AssertExtensions.Except( - fm => fm.DataSetVersionId, - fm => fm.Created, - fm => fm.Options, - fm => fm.OptionLinks - )); + Assert.All(testData.ExpectedFilters, (expectedFilter, filterIndex) => + { + var actualFilter = actualFilters[filterIndex]; - Assert.Equal(expectedFilter.Options.Count, actualFilter.Options.Count); - Assert.All(expectedFilter.Options, - (expectedOption, optionIndex) => - { - var actualOption = actualFilter.Options[optionIndex]; - actualOption.AssertDeepEqualTo(expectedOption, - notEqualProperties: AssertExtensions.Except( - o => o.Id, - o => o.Metas, - o => o.MetaLinks - )); + AssertFiltersEqual(expectedFilter, actualFilter); + AssertAllFilterOptionsEqual(expectedFilter, actualFilter); + }); + } - var actualOptionLink = actualOption.MetaLinks - .Single(link => link.MetaId == actualFilter.Id); + [Fact] + public async Task NextVersion_CorrectFilters_AutoMappedWithOldPublicIds() + { + var testData = ProcessorTestData.AbsenceSchool; - // Expect the PublicId to be encoded based on the sequence of option links inserted across all filters - Assert.Equal(SqidEncoder.Encode(++globalOptionIndex), actualOptionLink.PublicId); - }); - }); + var (sourceDataSetVersion, targetDataSetVersion, instanceId) = + await CreateDataSetInitialAndNextVersion( + nextVersionImportStage: DataSetVersionImportStage.ManualMapping, + nextVersionStatus: DataSetVersionStatus.Mapping); + + DataSetVersionMapping mapping = DataFixture.DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(sourceDataSetVersion.Id) + .WithTargetDataSetVersionId(targetDataSetVersion.Id) + .WithFilterMappingPlan( + DataFixture.FilterMappingPlanFromFilterMeta( + sourceFilters: testData.ExpectedFilters, + targetFilters: testData.ExpectedFilters + ) + ); + + foreach (var (_, filterMapping) in mapping.FilterMappingPlan.Mappings) + { + filterMapping.PublicId = $"{filterMapping.PublicId}-old"; + filterMapping.Source.Label = $"{filterMapping.Source.Label} old"; + } + + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ImportMetadata(testData, targetDataSetVersion, instanceId); + + await using var publicDataDbContext = GetDbContext(); + + var actualFilters = await GetDbFilterMetas(targetDataSetVersion.Id); + + Assert.Equal(testData.ExpectedFilters.Count, actualFilters.Count); + Assert.All(testData.ExpectedFilters, (expectedFilter, filterIndex) => + { + var actualFilter = actualFilters[filterIndex]; + + // Filter gets old public ID as it was auto mapped + expectedFilter.PublicId = $"{expectedFilter.PublicId}-old"; + + AssertFiltersEqual(expectedFilter, actualFilter); + AssertAllFilterOptionsEqual(expectedFilter, actualFilter); + }); } - [Theory] - [MemberData(nameof(TestDataFilesWithMetaInsertBatchSize))] - public async Task DataSetVersionMeta_CorrectFilters_WithMappings(ProcessorTestData testData, - int metaInsertBatchSize) + [Fact] + public async Task NextVersion_CorrectFilters_AutoNone() { + var testData = ProcessorTestData.AbsenceSchool; + var (sourceDataSetVersion, targetDataSetVersion, instanceId) = await CreateDataSetInitialAndNextVersion( nextVersionImportStage: DataSetVersionImportStage.ManualMapping, nextVersionStatus: DataSetVersionStatus.Mapping); - // In this test, we will create mappings for all the original filter options. - // 2 of these mappings will have candidates, and the rest will have no candidates - // mapped. + DataSetVersionMapping mapping = DataFixture.DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(sourceDataSetVersion.Id) + .WithTargetDataSetVersionId(targetDataSetVersion.Id) + .WithFilterMappingPlan( + DataFixture.FilterMappingPlanFromFilterMeta( + sourceFilters: testData.ExpectedFilters, + targetFilters: testData.ExpectedFilters + ) + ); + + foreach (var (_, filterMapping) in mapping.FilterMappingPlan.Mappings) + { + filterMapping.Type = MappingType.AutoNone; + filterMapping.PublicId = $"{filterMapping.PublicId}-old"; + filterMapping.Source.Label = $"{filterMapping.Source.Label} old"; + filterMapping.CandidateKey = null; + } - // Amend a couple of arbitrary mappings to identify some candidates. - var mappedOption1 = testData.ExpectedFilters.First().Options.First(); - var mappedOption2 = testData.ExpectedFilters.Last().Options.Last(); + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ImportMetadata(testData, targetDataSetVersion, instanceId); + + await using var publicDataDbContext = GetDbContext(); + + var actualFilters = await GetDbFilterMetas(targetDataSetVersion.Id); - var option1Mapping = new FilterOptionMapping + Assert.Equal(testData.ExpectedFilters.Count, actualFilters.Count); + Assert.All(testData.ExpectedFilters, (expectedFilter, filterIndex) => { - PublicId = "id-1", - Type = MappingType.AutoMapped, - CandidateKey = MappingKeyGenerators.FilterOptionMeta(mappedOption1), - Source = new MappableFilterOption { Label = "Option 1" } - }; + var actualFilter = actualFilters[filterIndex]; + + // Filter gets new public ID as it had no auto mapping + AssertFiltersEqual(expectedFilter, actualFilter); + AssertAllFilterOptionsEqual(expectedFilter, actualFilter); + }); + } + + [Fact] + public async Task NextVersion_CorrectFilters_ManualMapped() + { + var testData = ProcessorTestData.AbsenceSchool; + + var (sourceDataSetVersion, targetDataSetVersion, instanceId) = + await CreateDataSetInitialAndNextVersion( + nextVersionImportStage: DataSetVersionImportStage.ManualMapping, + nextVersionStatus: DataSetVersionStatus.Mapping); + + DataSetVersionMapping mapping = DataFixture.DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(sourceDataSetVersion.Id) + .WithTargetDataSetVersionId(targetDataSetVersion.Id) + .WithFilterMappingPlan( + DataFixture.FilterMappingPlanFromFilterMeta( + sourceFilters: testData.ExpectedFilters, + targetFilters: testData.ExpectedFilters + ) + ); + + foreach (var (_, filterMapping) in mapping.FilterMappingPlan.Mappings) + { + filterMapping.Type = MappingType.ManualMapped; + filterMapping.PublicId = $"{filterMapping.PublicId}-old"; + filterMapping.Source.Label = $"{filterMapping.Source.Label} old"; + } + + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ImportMetadata(testData, targetDataSetVersion, instanceId); + + await using var publicDataDbContext = GetDbContext(); - var option2Mapping = new FilterOptionMapping + var actualFilters = await GetDbFilterMetas(targetDataSetVersion.Id); + + Assert.Equal(testData.ExpectedFilters.Count, actualFilters.Count); + Assert.All(testData.ExpectedFilters, (expectedFilter, filterIndex) => { - PublicId = "id-2", - Type = MappingType.ManualMapped, - CandidateKey = MappingKeyGenerators.FilterOptionMeta(mappedOption2), - Source = new MappableFilterOption { Label = "Option 2" } - }; + var actualFilter = actualFilters[filterIndex]; + + // Filter gets old public ID as it was manually mapped + expectedFilter.PublicId = $"{expectedFilter.PublicId}-old"; + + AssertFiltersEqual(expectedFilter, actualFilter); + AssertAllFilterOptionsEqual(expectedFilter, actualFilter); + }); + } + + [Fact] + public async Task NextVersion_CorrectFilters_MixedMappings() + { + var testData = ProcessorTestData.AbsenceSchool; + + var (sourceDataSetVersion, targetDataSetVersion, instanceId) = + await CreateDataSetInitialAndNextVersion( + nextVersionImportStage: DataSetVersionImportStage.ManualMapping, + nextVersionStatus: DataSetVersionStatus.Mapping); + + DataSetVersionMapping mapping = DataFixture.DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(sourceDataSetVersion.Id) + .WithTargetDataSetVersionId(targetDataSetVersion.Id) + .WithFilterMappingPlan( + DataFixture.FilterMappingPlanFromFilterMeta( + sourceFilters: testData.ExpectedFilters, + targetFilters: testData.ExpectedFilters + ) + ); + + var academyTypeMapping = mapping.FilterMappingPlan.Mappings["academy_type"]; + + academyTypeMapping.Type = MappingType.AutoNone; + academyTypeMapping.PublicId = $"{academyTypeMapping.PublicId}-old"; + academyTypeMapping.Source.Label = $"{academyTypeMapping.Source.Label} old"; + academyTypeMapping.CandidateKey = null; + + var ncYearMapping = mapping.FilterMappingPlan.Mappings["ncyear"]; - var optionId = 0; + ncYearMapping.Type = MappingType.ManualMapped; + ncYearMapping.PublicId = $"{ncYearMapping.PublicId}-old"; + ncYearMapping.Source.Label = $"{ncYearMapping.Source.Label} old"; - var mappings = new DataSetVersionMapping + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ImportMetadata(testData, targetDataSetVersion, instanceId); + + await using var publicDataDbContext = GetDbContext(); + + var actualFilters = await GetDbFilterMetas(targetDataSetVersion.Id); + + Assert.Equal(testData.ExpectedFilters.Count, actualFilters.Count); + Assert.All(testData.ExpectedFilters, (expectedFilter, filterIndex) => { - SourceDataSetVersionId = sourceDataSetVersion.Id, - TargetDataSetVersionId = targetDataSetVersion.Id, - LocationMappingPlan = new LocationMappingPlan(), - FilterMappingPlan = new FilterMappingPlan + var actualFilter = actualFilters[filterIndex]; + + // Filter `ncyear` re-uses old public ID as it was manually mapped. + expectedFilter.PublicId = expectedFilter.Column == "ncyear" + ? $"{expectedFilter.PublicId}-old" + : expectedFilter.PublicId; + + AssertFiltersEqual(expectedFilter, actualFilter); + AssertAllFilterOptionsEqual(expectedFilter, actualFilter); + }); + } + + [Fact] + public async Task NextVersion_CorrectFilterOptions_AutoMappedWithSamePublicIds() + { + var testData = ProcessorTestData.AbsenceSchool; + + var (sourceDataSetVersion, targetDataSetVersion, instanceId) = + await CreateDataSetInitialAndNextVersion( + nextVersionImportStage: DataSetVersionImportStage.ManualMapping, + nextVersionStatus: DataSetVersionStatus.Mapping); + + DataSetVersionMapping mapping = DataFixture.DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(sourceDataSetVersion.Id) + .WithTargetDataSetVersionId(targetDataSetVersion.Id) + .WithFilterMappingPlan( + DataFixture.FilterMappingPlanFromFilterMeta( + sourceFilters: testData.ExpectedFilters, + targetFilters: testData.ExpectedFilters + ) + ); + + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ImportMetadata(testData, targetDataSetVersion, instanceId); + + var actualFilters = await GetDbFilterMetas(targetDataSetVersion.Id); + + Assert.Equal(testData.ExpectedFilters.Count, actualFilters.Count); + Assert.All(testData.ExpectedFilters, (expectedFilter, filterIndex) => + { + var actualFilter = actualFilters[filterIndex]; + + AssertFiltersEqual(expectedFilter, actualFilter); + AssertAllFilterOptionsEqual(expectedFilter, actualFilter); + }); + } + + [Fact] + public async Task NextVersion_CorrectFilterOptions_AutoMappedWithOldPublicIds() + { + var testData = ProcessorTestData.AbsenceSchool; + + var (sourceDataSetVersion, targetDataSetVersion, instanceId) = + await CreateDataSetInitialAndNextVersion( + nextVersionImportStage: DataSetVersionImportStage.ManualMapping, + nextVersionStatus: DataSetVersionStatus.Mapping); + + DataSetVersionMapping mapping = DataFixture.DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(sourceDataSetVersion.Id) + .WithTargetDataSetVersionId(targetDataSetVersion.Id) + .WithFilterMappingPlan( + DataFixture.FilterMappingPlanFromFilterMeta( + sourceFilters: testData.ExpectedFilters, + targetFilters: testData.ExpectedFilters + ) + ); + + foreach (var (_, filterMapping) in mapping.FilterMappingPlan.Mappings) + { + foreach (var (_, optionMapping) in filterMapping.OptionMappings) { - Mappings = testData - .ExpectedFilters - .ToDictionary( - keySelector: MappingKeyGenerators.Filter, - elementSelector: filter => new FilterMapping - { - Type = MappingType.AutoMapped, - Source = new MappableFilter { Label = filter.Label }, - PublicId = filter.PublicId, - OptionMappings = filter - .Options - .ToDictionary( - keySelector: MappingKeyGenerators.FilterOptionMeta, - elementSelector: option => - { - optionId++; - - return option == mappedOption1 ? option1Mapping - : option == mappedOption2 ? option2Mapping - : new FilterOptionMapping - { - PublicId = optionId.ToString(), - Type = optionId % 2 == 0 - ? MappingType.AutoNone - : MappingType.ManualNone, - Source = new MappableFilterOption - { - Label = option.Label - }, - }; - } - ) - }) + optionMapping.PublicId = $"{optionMapping.PublicId}-old"; + optionMapping.Source.Label = $"{optionMapping.Source.Label} old"; } - }; + } - await AddTestData(context => + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ImportMetadata(testData, targetDataSetVersion, instanceId); + + var actualFilters = await GetDbFilterMetas(targetDataSetVersion.Id); + + Assert.Equal(testData.ExpectedFilters.Count, actualFilters.Count); + Assert.All(testData.ExpectedFilters, (expectedFilter, filterIndex) => { - context.DataSetVersionMappings.Add(mappings); + var actualFilter = actualFilters[filterIndex]; + + AssertFiltersEqual(expectedFilter, actualFilter); + + // Filter options re-use old public IDs as they were auto mapped + foreach (var optionLink in expectedFilter.OptionLinks) + { + optionLink.PublicId = $"{optionLink.PublicId}-old"; + } + + AssertAllFilterOptionsEqual(expectedFilter, actualFilter); }); + } - await ImportMetadata(testData, targetDataSetVersion, instanceId, metaInsertBatchSize); + [Fact] + public async Task NextVersion_CorrectFilterOptions_AutoNoneOptions() + { + var testData = ProcessorTestData.AbsenceSchool; - await using var publicDataDbContext = GetDbContext(); + var (sourceDataSetVersion, targetDataSetVersion, instanceId) = + await CreateDataSetInitialAndNextVersion( + nextVersionImportStage: DataSetVersionImportStage.ManualMapping, + nextVersionStatus: DataSetVersionStatus.Mapping); - var actualFilters = await publicDataDbContext.FilterMetas - .Include(fm => fm.Options.OrderBy(o => o.Label)) - .ThenInclude(fom => fom.MetaLinks) - .Where(fm => fm.DataSetVersionId == targetDataSetVersion.Id) - .OrderBy(fm => fm.Label) - .ToListAsync(); + DataSetVersionMapping mapping = DataFixture.DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(sourceDataSetVersion.Id) + .WithTargetDataSetVersionId(targetDataSetVersion.Id) + .WithFilterMappingPlan( + DataFixture.FilterMappingPlanFromFilterMeta( + sourceFilters: testData.ExpectedFilters, + targetFilters: testData.ExpectedFilters + ) + ); + + foreach (var (_, filterMapping) in mapping.FilterMappingPlan.Mappings) + { + foreach (var (_, optionMapping) in filterMapping.OptionMappings) + { + optionMapping.Type = MappingType.AutoNone; + optionMapping.PublicId = $"{optionMapping.PublicId}-old"; + optionMapping.Source.Label = $"{optionMapping.Source.Label} old"; + optionMapping.CandidateKey = null; + } + } - var globalOptionIndex = 0; + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ImportMetadata(testData, targetDataSetVersion, instanceId); + + var actualFilters = await GetDbFilterMetas(targetDataSetVersion.Id); Assert.Equal(testData.ExpectedFilters.Count, actualFilters.Count); - Assert.All(testData.ExpectedFilters, - (expectedFilter, index) => + Assert.All(testData.ExpectedFilters, (expectedFilter, filterIndex) => + { + var actualFilter = actualFilters[filterIndex]; + + AssertFiltersEqual(expectedFilter, actualFilter); + + // Filter options get new public ID as it they have no auto mapping + AssertAllFilterOptionsEqual(expectedFilter, actualFilter); + }); + } + + [Fact] + public async Task NextVersion_CorrectFilterOptions_ManualMappedOptions() + { + var testData = ProcessorTestData.AbsenceSchool; + + var (sourceDataSetVersion, targetDataSetVersion, instanceId) = + await CreateDataSetInitialAndNextVersion( + nextVersionImportStage: DataSetVersionImportStage.ManualMapping, + nextVersionStatus: DataSetVersionStatus.Mapping); + + DataSetVersionMapping mapping = DataFixture.DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(sourceDataSetVersion.Id) + .WithTargetDataSetVersionId(targetDataSetVersion.Id) + .WithFilterMappingPlan( + DataFixture.FilterMappingPlanFromFilterMeta( + sourceFilters: testData.ExpectedFilters, + targetFilters: testData.ExpectedFilters + ) + ); + + foreach (var (_, filterMapping) in mapping.FilterMappingPlan.Mappings) + { + foreach (var (_, optionMapping) in filterMapping.OptionMappings) { - var actualFilter = actualFilters[index]; - actualFilter.AssertDeepEqualTo(expectedFilter, - notEqualProperties: AssertExtensions.Except( - fm => fm.DataSetVersionId, - fm => fm.Created, - fm => fm.Options, - fm => fm.OptionLinks - )); + optionMapping.Type = MappingType.ManualMapped; + optionMapping.PublicId = $"{optionMapping.PublicId}-old"; + optionMapping.Source.Label = $"{optionMapping.Source.Label} old"; + } + } - Assert.Equal(expectedFilter.Options.Count, actualFilter.Options.Count); - Assert.All(expectedFilter.Options, - (expectedOption, optionIndex) => - { - var actualOption = actualFilter.Options[optionIndex]; - actualOption.AssertDeepEqualTo(expectedOption, - notEqualProperties: AssertExtensions.Except( - o => o.Id, - o => o.Metas, - o => o.MetaLinks - )); + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); - var actualOptionLink = actualOption.MetaLinks - .Single(link => link.MetaId == actualFilter.Id); - - // If the filter option link is related to either filter option that was mapped to - // a candidate in the mappings, expect the PublicId to have been carried over from - // the source filter option to the new filter option to allow backwards-compatibility - // with queries that use the source filter option's PublicId. - if (actualOptionLink.Option.Label == mappedOption1.Label) - { - Assert.Equal("id-1", actualOptionLink.PublicId); - } - else if (actualOptionLink.Option.Label == mappedOption2.Label) - { - Assert.Equal("id-2", actualOptionLink.PublicId); - } - else - { - // Otherwise expect the PublicId to be encoded based on the sequence of option links - // inserted across all filters, save for those that have been allocated their PublicId - // based upon the mappings. - Assert.Equal(SqidEncoder.Encode(++globalOptionIndex), actualOptionLink.PublicId); - } - }); - }); + await ImportMetadata(testData, targetDataSetVersion, instanceId); + + var actualFilters = await GetDbFilterMetas(targetDataSetVersion.Id); + + Assert.Equal(testData.ExpectedFilters.Count, actualFilters.Count); + Assert.All(testData.ExpectedFilters, (expectedFilter, filterIndex) => + { + var actualFilter = actualFilters[filterIndex]; + + AssertFiltersEqual(expectedFilter, actualFilter); + + // Filter options re-use old public IDs as they were manually mapped + foreach (var optionLink in expectedFilter.OptionLinks) + { + optionLink.PublicId = $"{optionLink.PublicId}-old"; + } + + AssertAllFilterOptionsEqual(expectedFilter, actualFilter); + }); } - [Theory] - [MemberData(nameof(TestDataFiles))] - public async Task DataSetVersionMeta_CorrectIndicators(ProcessorTestData testData) + [Fact] + public async Task NextVersion_CorrectFilterOptions_MixedOptionMappings() { + var testData = ProcessorTestData.AbsenceSchool; + + var (sourceDataSetVersion, targetDataSetVersion, instanceId) = + await CreateDataSetInitialAndNextVersion( + nextVersionImportStage: DataSetVersionImportStage.ManualMapping, + nextVersionStatus: DataSetVersionStatus.Mapping); + + DataSetVersionMapping mapping = DataFixture.DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(sourceDataSetVersion.Id) + .WithTargetDataSetVersionId(targetDataSetVersion.Id) + .WithFilterMappingPlan( + DataFixture.FilterMappingPlanFromFilterMeta( + sourceFilters: testData.ExpectedFilters, + targetFilters: testData.ExpectedFilters + ) + ); + + foreach (var (_, optionMapping) in mapping.FilterMappingPlan.Mappings["academy_type"].OptionMappings) + { + optionMapping.Type = MappingType.AutoNone; + optionMapping.PublicId = $"{optionMapping.PublicId}-old"; + optionMapping.Source.Label = $"{optionMapping.Source.Label} old"; + optionMapping.CandidateKey = null; + } + + foreach (var (_, optionMapping) in mapping.FilterMappingPlan.Mappings["ncyear"].OptionMappings) + { + optionMapping.Type = MappingType.ManualMapped; + optionMapping.PublicId = $"{optionMapping.PublicId}-old"; + optionMapping.Source.Label = $"{optionMapping.Source.Label} old"; + } + + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ImportMetadata(testData, targetDataSetVersion, instanceId); + + await using var publicDataDbContext = GetDbContext(); + + var actualFilters = await GetDbFilterMetas(targetDataSetVersion.Id); + + Assert.Equal(testData.ExpectedFilters.Count, actualFilters.Count); + Assert.All(testData.ExpectedFilters, (expectedFilter, filterIndex) => + { + var actualFilter = actualFilters[filterIndex]; + + AssertFiltersEqual(expectedFilter, actualFilter); + + // Only `ncyear` options re-use old public IDs as they were manually mapped. + foreach (var optionLink in expectedFilter.OptionLinks) + { + optionLink.PublicId = expectedFilter.Column == "ncyear" + ? $"{optionLink.PublicId}-old" + : optionLink.PublicId; + } + + AssertAllFilterOptionsEqual(expectedFilter, actualFilter); + }); + } + + [Fact] + public async Task InitialVersion_CorrectIndicators() + { + var testData = ProcessorTestData.AbsenceSchool; + var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); await ImportMetadata(testData, dataSetVersion, instanceId); await using var publicDataDbContext = GetDbContext(); - var actualIndicators = await publicDataDbContext.IndicatorMetas - .Where(im => im.DataSetVersionId == dataSetVersion.Id) - .OrderBy(im => im.Label) - .ToListAsync(); + var actualIndicators = await GetDbIndicatorMetas(dataSetVersion.Id); Assert.Equal(testData.ExpectedIndicators.Count, actualIndicators.Count); - Assert.All(testData.ExpectedIndicators, - (expectedIndicator, index) => + Assert.All(testData.ExpectedIndicators, (expectedIndicator, index) => + { + var actualIndicator = actualIndicators[index]; + + actualIndicator.AssertDeepEqualTo( + expectedIndicator, + notEqualProperties: AssertExtensions.Except( + m => m.DataSetVersionId, + m => m.Created + )); + }); + } + + [Fact] + public async Task NextVersion_CorrectIndicators_OldPublicIds() + { + var testData = ProcessorTestData.AbsenceSchool; + + var existingIndicators = testData.ExpectedIndicators + .Select(i => { - var actualIndicator = actualIndicators[index]; - actualIndicator.AssertDeepEqualTo(expectedIndicator, - notEqualProperties: AssertExtensions.Except( - im => im.DataSetVersionId, - im => im.Created - )); - }); + var indicator = i.ShallowClone(); + + indicator.Id = 0; + indicator.PublicId = $"{indicator.PublicId}-old"; + + return indicator; + }) + .ToList(); + + var (sourceDataSetVersion, targetDataSetVersion, instanceId) = + await CreateDataSetInitialAndNextVersion( + initialVersionMeta: new DataSetVersionMeta + { + IndicatorMetas = existingIndicators + }, + nextVersionImportStage: DataSetVersionImportStage.ManualMapping, + nextVersionStatus: DataSetVersionStatus.Mapping); + + DataSetVersionMapping mapping = DataFixture.DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(sourceDataSetVersion.Id) + .WithTargetDataSetVersionId(targetDataSetVersion.Id); + + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ImportMetadata(testData, targetDataSetVersion, instanceId); + + var actualIndicators = await GetDbIndicatorMetas(targetDataSetVersion.Id); + + Assert.Equal(testData.ExpectedIndicators.Count, actualIndicators.Count); + Assert.All(testData.ExpectedIndicators, (expectedIndicator, index) => + { + var actualIndicator = actualIndicators[index]; + + actualIndicator.AssertDeepEqualTo( + expectedIndicator, + notEqualProperties: AssertExtensions.Except( + m => m.Id, + m => m.PublicId, + m => m.DataSetVersionId, + m => m.Created + )); + + + // Indicators re-use old public IDs of existing indicators. + Assert.Equal($"{expectedIndicator.PublicId}-old", actualIndicator.PublicId); + }); + } + + [Fact] + public async Task NextVersion_CorrectIndicators_NewPublicIds() + { + var testData = ProcessorTestData.AbsenceSchool; + + // Set up indicators that are nothing like the test data set's indicators. + var existingIndicators = DataFixture + .DefaultIndicatorMeta() + .GenerateList(3); + + var (sourceDataSetVersion, targetDataSetVersion, instanceId) = + await CreateDataSetInitialAndNextVersion( + initialVersionMeta: new DataSetVersionMeta + { + IndicatorMetas = existingIndicators + }, + nextVersionImportStage: DataSetVersionImportStage.ManualMapping, + nextVersionStatus: DataSetVersionStatus.Mapping); + + DataSetVersionMapping mapping = DataFixture.DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(sourceDataSetVersion.Id) + .WithTargetDataSetVersionId(targetDataSetVersion.Id); + + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ImportMetadata(testData, targetDataSetVersion, instanceId); + + var actualIndicators = await GetDbIndicatorMetas(targetDataSetVersion.Id); + + Assert.Equal(testData.ExpectedIndicators.Count, actualIndicators.Count); + Assert.All(testData.ExpectedIndicators, (expectedIndicator, index) => + { + var actualIndicator = actualIndicators[index]; + + actualIndicator.AssertDeepEqualTo( + expectedIndicator, + notEqualProperties: AssertExtensions.Except( + m => m.Id, + m => m.PublicId, + m => m.DataSetVersionId, + m => m.Created + )); + + // Indicators get new public IDs offset by number of existing indicators. + Assert.Equal( + SqidEncoder.Encode(expectedIndicator.Id + existingIndicators.Count), + actualIndicator.PublicId + ); + }); + } + + [Fact] + public async Task NextVersion_CorrectIndicators_NewPublicIdsWhenColumnsChange() + { + var testData = ProcessorTestData.AbsenceSchool; + + // Set up existing indicators that only differ in column names. Currently, + // indicators can't be mapped so the new indicators must be given new public IDs. + var existingIndicators = testData.ExpectedIndicators + .Select(i => + { + var indicator = i.ShallowClone(); + + indicator.Id = 0; + indicator.Column = $"{indicator.Column}_old"; + + return indicator; + }) + .ToList(); + + var (sourceDataSetVersion, targetDataSetVersion, instanceId) = + await CreateDataSetInitialAndNextVersion( + initialVersionMeta: new DataSetVersionMeta + { + IndicatorMetas = existingIndicators + }, + nextVersionImportStage: DataSetVersionImportStage.ManualMapping, + nextVersionStatus: DataSetVersionStatus.Mapping); + + DataSetVersionMapping mapping = DataFixture.DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(sourceDataSetVersion.Id) + .WithTargetDataSetVersionId(targetDataSetVersion.Id); + + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ImportMetadata(testData, targetDataSetVersion, instanceId); + + var actualIndicators = await GetDbIndicatorMetas(targetDataSetVersion.Id); + + Assert.Equal(testData.ExpectedIndicators.Count, actualIndicators.Count); + Assert.All(testData.ExpectedIndicators, (expectedIndicator, index) => + { + var actualIndicator = actualIndicators[index]; + + actualIndicator.AssertDeepEqualTo( + expectedIndicator, + notEqualProperties: AssertExtensions.Except( + m => m.Id, + m => m.PublicId, + m => m.DataSetVersionId, + m => m.Created + )); + + // Indicators get new public IDs offset by number of existing indicators. + Assert.Equal( + SqidEncoder.Encode(expectedIndicator.Id + existingIndicators.Count), + actualIndicator.PublicId + ); + }); } + [Fact] + public async Task NextVersion_CorrectIndicators_MixtureHaveCorrectPublicIds() + { + var testData = ProcessorTestData.AbsenceSchool; + + List existingIndicators = + [ + .. testData.ExpectedIndicators + .Select(i => + { + var indicator = i.ShallowClone(); + + indicator.Id = 0; + + return indicator; + }), + .. DataFixture.DefaultIndicatorMeta().Generate(1) + ]; + + existingIndicators[2].PublicId = $"{existingIndicators[2].PublicId}-old"; + existingIndicators[3].PublicId = $"{existingIndicators[3].PublicId}-old"; + + existingIndicators[4].Column = $"{existingIndicators[4].PublicId}_old"; + + var (sourceDataSetVersion, targetDataSetVersion, instanceId) = + await CreateDataSetInitialAndNextVersion( + initialVersionMeta: new DataSetVersionMeta + { + IndicatorMetas = existingIndicators + }, + nextVersionImportStage: DataSetVersionImportStage.ManualMapping, + nextVersionStatus: DataSetVersionStatus.Mapping); + + DataSetVersionMapping mapping = DataFixture.DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(sourceDataSetVersion.Id) + .WithTargetDataSetVersionId(targetDataSetVersion.Id); + + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ImportMetadata(testData, targetDataSetVersion, instanceId); + + var actualIndicators = await GetDbIndicatorMetas(targetDataSetVersion.Id); + + Assert.Equal(testData.ExpectedIndicators.Count, actualIndicators.Count); + + Assert.Equal(testData.ExpectedIndicators[0].PublicId, actualIndicators[0].PublicId); + Assert.Equal(testData.ExpectedIndicators[1].PublicId, actualIndicators[1].PublicId); + Assert.Equal($"{testData.ExpectedIndicators[2].PublicId}-old", actualIndicators[2].PublicId); + Assert.Equal($"{testData.ExpectedIndicators[3].PublicId}-old", actualIndicators[3].PublicId); + Assert.Equal( + SqidEncoder.Encode(testData.ExpectedIndicators[4].Id + existingIndicators.Count), + actualIndicators[4].PublicId + ); + } + } + + public class ImportMetadataDuckDbTests(ProcessorFunctionsIntegrationTestFixture fixture) + : ImportMetadataFunctionTests(fixture) + { [Theory] [MemberData(nameof(TestDataFiles))] - public async Task DuckDbMeta_CorrectLocationOptions(ProcessorTestData testData) + public async Task InitialVersion_CorrectLocationOptions(ProcessorTestData testData) { var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); @@ -583,7 +1119,7 @@ public async Task DuckDbMeta_CorrectLocationOptions(ProcessorTestData testData) [Theory] [MemberData(nameof(TestDataFiles))] - public async Task DuckDbMeta_CorrectTimePeriods(ProcessorTestData testData) + public async Task InitialVersion_CorrectTimePeriods(ProcessorTestData testData) { var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); @@ -612,7 +1148,7 @@ public async Task DuckDbMeta_CorrectTimePeriods(ProcessorTestData testData) [Theory] [MemberData(nameof(TestDataFiles))] - public async Task DuckDbMeta_CorrectFilterOptions(ProcessorTestData testData) + public async Task InitialVersion_CorrectFilterOptions(ProcessorTestData testData) { var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); @@ -657,7 +1193,7 @@ public async Task DuckDbMeta_CorrectFilterOptions(ProcessorTestData testData) [Theory] [MemberData(nameof(TestDataFiles))] - public async Task DuckDbMeta_CorrectIndicators(ProcessorTestData testData) + public async Task InitialVersion_CorrectIndicators(ProcessorTestData testData) { var (dataSetVersion, instanceId) = await CreateDataSetInitialVersion(Stage.PreviousStage()); @@ -693,24 +1229,68 @@ public async Task DuckDbMeta_CorrectIndicators(ProcessorTestData testData) } }); } + } - private async Task ImportMetadata( - ProcessorTestData testData, - DataSetVersion dataSetVersion, - Guid instanceId, - int? metaInsertBatchSize = null) + private async Task ImportMetadata( + ProcessorTestData testData, + DataSetVersion dataSetVersion, + Guid instanceId, + int? metaInsertBatchSize = null) + { + SetupCsvDataFilesForDataSetVersion(testData, dataSetVersion); + + // Override default app settings if provided + if (metaInsertBatchSize.HasValue) { - SetupCsvDataFilesForDataSetVersion(testData, dataSetVersion); + var appOptions = GetRequiredService>(); + appOptions.Value.MetaInsertBatchSize = metaInsertBatchSize.Value; + } - // Override default app settings if provided - if (metaInsertBatchSize.HasValue) - { - var appOptions = GetRequiredService>(); - appOptions.Value.MetaInsertBatchSize = metaInsertBatchSize.Value; - } + var function = GetRequiredService(); + await function.ImportMetadata(instanceId, CancellationToken.None); + } - var function = GetRequiredService(); - await function.ImportMetadata(instanceId, CancellationToken.None); - } + private async Task> GetDbFilterMetas(Guid dataSetVersionId) + => await GetDbContext().FilterMetas + .Include(fm => fm.Options.OrderBy(o => o.Label)) + .Include(fm => fm.OptionLinks.OrderBy(l => l.Option.Label)) + .ThenInclude(fm => fm.Option) + .Where(fm => fm.DataSetVersionId == dataSetVersionId) + .OrderBy(fm => fm.Label) + .ToListAsync(); + + private Task> GetDbIndicatorMetas(Guid dataSetVersionId) + => GetDbContext().IndicatorMetas + .Where(im => im.DataSetVersionId == dataSetVersionId) + .OrderBy(im => im.Label) + .ToListAsync(); + + private static void AssertAllFilterOptionsEqual(FilterMeta expectedFilter, FilterMeta actualFilter) + { + Assert.Equal(expectedFilter.Options.Count, actualFilter.Options.Count); + Assert.Equal(expectedFilter.OptionLinks.Count, actualFilter.OptionLinks.Count); + + Assert.All(expectedFilter.OptionLinks, (expectedOptionLink, linkIndex) => + { + var actualOptionLink = actualFilter.OptionLinks[linkIndex]; + + Assert.Equal(expectedOptionLink.PublicId, actualOptionLink.PublicId); + + var expectedOption = expectedFilter.Options[linkIndex]; + + Assert.Equal(expectedOption.Label, actualOptionLink.Option.Label); + Assert.Equal(expectedOption.IsAggregate, actualOptionLink.Option.IsAggregate); + }); + } + + private static void AssertFiltersEqual(FilterMeta expectedFilter, FilterMeta actualFilter) + { + actualFilter.AssertDeepEqualTo(expectedFilter, + notEqualProperties: AssertExtensions.Except( + m => m.DataSetVersionId, + m => m.Options, + m => m.OptionLinks, + m => m.Created + )); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorTestData.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorTestData.cs index 91e1e06b77d..1c94e352188 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorTestData.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorTestData.cs @@ -218,17 +218,42 @@ public record ProcessorTestData [ new FilterOptionMeta { + Id = 1, Label = "Primary sponsor led academy", }, new FilterOptionMeta { + Id = 2, Label = "Secondary free school", }, new FilterOptionMeta { + Id = 3, Label = "Secondary sponsor led academy", }, ], + OptionLinks = + [ + new FilterOptionMetaLink + { + MetaId = 1, + OptionId = 1, + PublicId = SqidEncoder.Encode(1), + }, + new FilterOptionMetaLink + { + MetaId = 1, + OptionId = 2, + PublicId = SqidEncoder.Encode(2), + }, + new FilterOptionMetaLink + { + + MetaId = 1, + OptionId = 3, + PublicId = SqidEncoder.Encode(3), + } + ], }, new FilterMeta { @@ -242,21 +267,53 @@ public record ProcessorTestData [ new FilterOptionMeta { + Id = 4, Label = "Year 10", }, new FilterOptionMeta { + Id = 5, Label = "Year 4", }, new FilterOptionMeta { + Id = 6, Label = "Year 6", }, new FilterOptionMeta { + Id = 7, Label = "Year 8", }, ], + OptionLinks = + [ + new FilterOptionMetaLink + { + MetaId = 2, + OptionId = 4, + PublicId = SqidEncoder.Encode(4), + }, + new FilterOptionMetaLink + { + MetaId = 2, + OptionId = 5, + PublicId = SqidEncoder.Encode(5), + }, + new FilterOptionMetaLink + { + + MetaId = 2, + OptionId = 6, + PublicId = SqidEncoder.Encode(6), + }, + new FilterOptionMetaLink + { + MetaId = 2, + OptionId = 7, + PublicId = SqidEncoder.Encode(7), + } + ], }, new FilterMeta { @@ -270,18 +327,42 @@ public record ProcessorTestData [ new FilterOptionMeta { + Id = 8, Label = "State-funded primary", }, new FilterOptionMeta { + Id = 9, Label = "State-funded secondary", }, new FilterOptionMeta { + Id = 10, Label = "Total", IsAggregate = true }, ], + OptionLinks = + [ + new FilterOptionMetaLink + { + MetaId = 3, + OptionId = 8, + PublicId = SqidEncoder.Encode(8), + }, + new FilterOptionMetaLink + { + MetaId = 3, + OptionId = 9, + PublicId = SqidEncoder.Encode(9), + }, + new FilterOptionMetaLink + { + MetaId = 3, + OptionId = 10, + PublicId = SqidEncoder.Encode(10), + } + ] }, ], ExpectedIndicators = diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterMetaRepository.cs index 375af469b5a..89a29772b64 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterMetaRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterMetaRepository.cs @@ -32,6 +32,7 @@ public async Task>> ReadFilterMet duckDbConnection, dataSetVersion, allowedColumns, + publicIdMappings: [], cancellationToken); return await metas @@ -53,17 +54,18 @@ public async Task CreateFilterMetas( IReadOnlySet allowedColumns, CancellationToken cancellationToken = default) { + var publicIdMappings = await CreatePublicIdMappings(dataSetVersion, cancellationToken); + var metas = await GetFilterMetas( duckDbConnection, dataSetVersion, allowedColumns, + publicIdMappings.Filters, cancellationToken); publicDataDbContext.FilterMetas.AddRange(metas); await publicDataDbContext.SaveChangesAsync(cancellationToken); - var publicIdMappings = await CreatePublicIdMappings(dataSetVersion, cancellationToken); - foreach (var meta in metas) { var options = await GetFilterOptionMeta( @@ -80,16 +82,8 @@ await optionTable .Merge() .Using(options) .On( - o => new - { - o.Label, - o.IsAggregate - }, - o => new - { - o.Label, - o.IsAggregate - } + o => new { o.Label, o.IsAggregate }, + o => new { o.Label, o.IsAggregate } ) .InsertWhenNotMatched() .MergeAsync(cancellationToken); @@ -150,11 +144,17 @@ await optionTable $"Inserted: {insertedLinks}, expected: {options.Count}"); } - // Increase the sequence only by the amount that we used to generate new PublicIds. - await publicDataDbContext.SetSequenceValue( - PublicDataDbContext.FilterOptionMetaLinkSequence, - currentId - 1, - cancellationToken); + // Avoid trying to set to 0 (which only + // happens synthetically during tests). + if (currentId > 1) + { + // Increase the sequence only by the amount that we used to generate new PublicIds. + await publicDataDbContext.SetSequenceValue( + PublicDataDbContext.FilterOptionMetaLinkSequence, + currentId - 1, + cancellationToken + ); + } } } @@ -162,21 +162,23 @@ private async Task> GetFilterMetas( IDuckDbConnection duckDbConnection, DataSetVersion dataSetVersion, IReadOnlySet allowedColumns, + Dictionary publicIdMappings, CancellationToken cancellationToken) { var currentId = await publicDataDbContext.NextSequenceValue( PublicDataDbContext.FilterMetasIdSequence, cancellationToken); - var metas = (await duckDbConnection.SqlBuilder( - $""" - SELECT * - FROM '{dataSetVersionPathResolver.CsvMetadataPath(dataSetVersion):raw}' - WHERE "col_type" = {MetaFileRow.ColumnType.Filter.ToString()} - AND "col_name" IN ({allowedColumns}) - """) - .QueryAsync(cancellationToken: cancellationToken) - ) + var metaRows = await duckDbConnection.SqlBuilder( + $""" + SELECT * + FROM '{dataSetVersionPathResolver.CsvMetadataPath(dataSetVersion):raw}' + WHERE "col_type" = {MetaFileRow.ColumnType.Filter.ToString()} + AND "col_name" IN ({allowedColumns}) + """) + .QueryAsync(cancellationToken: cancellationToken); + + var metas = metaRows .OrderBy(row => row.Label) .Select(row => { @@ -185,7 +187,7 @@ private async Task> GetFilterMetas( return new FilterMeta { Id = id, - PublicId = SqidEncoder.Encode(id), + PublicId = publicIdMappings.GetValueOrDefault(row.ColName, SqidEncoder.Encode(id)), Column = row.ColName, DataSetVersionId = dataSetVersion.Id, Label = row.Label, @@ -194,10 +196,16 @@ private async Task> GetFilterMetas( }) .ToList(); - await publicDataDbContext.SetSequenceValue( - PublicDataDbContext.FilterMetasIdSequence, - currentId - 1, - cancellationToken); + // Avoid trying to set to 0 (which only + // happens synthetically during tests). + if (currentId > 1) + { + await publicDataDbContext.SetSequenceValue( + PublicDataDbContext.FilterMetasIdSequence, + currentId - 1, + cancellationToken + ); + } return metas; } @@ -230,32 +238,40 @@ private async Task CreatePublicIdMappings( DataSetVersion dataSetVersion, CancellationToken cancellationToken) { - var mappings = await EntityFrameworkQueryableExtensions - .SingleOrDefaultAsync(publicDataDbContext - .DataSetVersionMappings, - mapping => mapping.TargetDataSetVersionId == dataSetVersion.Id, - cancellationToken); + var mappings = await publicDataDbContext.DataSetVersionMappings + .Where(mapping => mapping.TargetDataSetVersionId == dataSetVersion.Id) + .SingleOrDefaultAsync(token: cancellationToken); if (mappings is null) { return new PublicIdMappings(); } - var mappingsByFilter = mappings - .FilterMappingPlan + var filterMappings = mappings.FilterMappingPlan + .Mappings + .Where(mapping => mapping.Value.Type is MappingType.AutoMapped or MappingType.ManualMapped) + .ToDictionary( + filter => filter.Key, + filter => filter.Value.PublicId); + + var filterOptionMappings = mappings.FilterMappingPlan .Mappings .ToDictionary( - keySelector: filter => filter.Key, - elementSelector: filter => filter - .Value + filter => filter.Key, + filter => filter.Value .OptionMappings .Values .Where(mapping => mapping.Type is MappingType.AutoMapped or MappingType.ManualMapped) .ToDictionary( - keySelector: mapping => mapping.CandidateKey!, - elementSelector: mapping => mapping.PublicId)); + mapping => mapping.CandidateKey!, + mapping => mapping.PublicId) + ); - return new PublicIdMappings { Filters = mappingsByFilter }; + return new PublicIdMappings + { + Filters = filterMappings, + FilterOptions = filterOptionMappings, + }; } private static string CreatePublicIdForFilterOptionMetaLink( @@ -273,11 +289,19 @@ private static string CreatePublicIdForFilterOptionMetaLink( private record PublicIdMappings { - public Dictionary> Filters { get; init; } = []; + /// + /// Filter public IDs mappings by column. + /// + public Dictionary Filters { get; init; } = []; + + /// + /// Filter option public ID mappings grouped by filter column. + /// + public Dictionary> FilterOptions { get; init; } = []; public string? GetPublicIdForFilterOptionCandidate(string filterKey, string filterOptionCandidateKey) { - if (!Filters.TryGetValue(filterKey, out var filterOptionMappings)) + if (!FilterOptions.TryGetValue(filterKey, out var filterOptionMappings)) { return null; } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorMetaRepository.cs index 62578abefaf..40f19745822 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorMetaRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorMetaRepository.cs @@ -6,6 +6,7 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Utils; using InterpolatedSql.Dapper; +using Microsoft.EntityFrameworkCore; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository; @@ -19,19 +20,32 @@ public async Task CreateIndicatorMetas( IReadOnlySet allowedColumns, CancellationToken cancellationToken = default) { + var sourceDataSetVersionId = await GetSourceDataSetVersionId(dataSetVersion, cancellationToken); + + var existingMetaIdsByColumn = sourceDataSetVersionId is not null + ? await publicDataDbContext.IndicatorMetas + .Where(meta => meta.DataSetVersionId == sourceDataSetVersionId) + .ToDictionaryAsync( + meta => meta.Column, + meta => meta.PublicId, + cancellationToken + ) + : []; + var currentId = await publicDataDbContext.NextSequenceValue( PublicDataDbContext.IndicatorMetasIdSequence, cancellationToken); - var metas = (await duckDbConnection.SqlBuilder( - $""" - SELECT * - FROM '{dataSetVersionPathResolver.CsvMetadataPath(dataSetVersion):raw}' - WHERE "col_type" = {MetaFileRow.ColumnType.Indicator.ToString()} - AND "col_name" IN ({allowedColumns}) - """) - .QueryAsync(cancellationToken: cancellationToken) - ) + var metaRows = await duckDbConnection.SqlBuilder( + $""" + SELECT * + FROM '{dataSetVersionPathResolver.CsvMetadataPath(dataSetVersion):raw}' + WHERE "col_type" = {MetaFileRow.ColumnType.Indicator.ToString()} + AND "col_name" IN ({allowedColumns}) + """) + .QueryAsync(cancellationToken: cancellationToken); + + var metas = metaRows .OrderBy(row => row.Label) .Select(row => { @@ -41,7 +55,7 @@ public async Task CreateIndicatorMetas( { Id = id, DataSetVersionId = dataSetVersion.Id, - PublicId = SqidEncoder.Encode(id), + PublicId = existingMetaIdsByColumn.GetValueOrDefault(row.ColName, SqidEncoder.Encode(id)), Column = row.ColName, Label = row.Label, Unit = row.ParsedIndicatorUnit, @@ -59,4 +73,14 @@ await publicDataDbContext.SetSequenceValue( currentId - 1, cancellationToken); } + + private async Task GetSourceDataSetVersionId( + DataSetVersion dataSetVersion, + CancellationToken cancellationToken) + { + return await publicDataDbContext.DataSetVersionMappings + .Where(mapping => mapping.TargetDataSetVersionId == dataSetVersion.Id) + .Select(mapping => mapping.SourceDataSetVersionId) + .SingleOrDefaultAsync(cancellationToken); + } } From c35c6be6387acbc682e9411c3530bfe826f6d44b Mon Sep 17 00:00:00 2001 From: Amy Benson Date: Thu, 10 Oct 2024 13:36:15 +0100 Subject: [PATCH 47/80] EES-5306 fix high contrast style for search form --- .../src/components/form/FormSearchBar.module.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/explore-education-statistics-common/src/components/form/FormSearchBar.module.scss b/src/explore-education-statistics-common/src/components/form/FormSearchBar.module.scss index 013238ccdc8..863195ef741 100644 --- a/src/explore-education-statistics-common/src/components/form/FormSearchBar.module.scss +++ b/src/explore-education-statistics-common/src/components/form/FormSearchBar.module.scss @@ -2,7 +2,7 @@ .button { background: $govuk-brand-colour; - border: 0; + border: 2px solid transparent; color: govuk-colour('white'); cursor: pointer; height: 40px; From e084c01eeeecb8c8f20fc6f9c19f7c8f7636e989 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Thu, 10 Oct 2024 14:18:47 +0100 Subject: [PATCH 48/80] EES-5560 - added guidance notes to public data set changelog section --- .../components/ApiDataSetChangelog.tsx | 2 +- .../data-catalogue/DataSetFilePage.tsx | 3 +- .../components/DataSetFileApiChangelog.tsx | 10 +++++- .../DataSetFileApiChangelog.test.tsx | 35 ++++++++++++++++++- 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/src/explore-education-statistics-common/src/modules/data-catalogue/components/ApiDataSetChangelog.tsx b/src/explore-education-statistics-common/src/modules/data-catalogue/components/ApiDataSetChangelog.tsx index 344cfd21b0e..54a765f07a5 100644 --- a/src/explore-education-statistics-common/src/modules/data-catalogue/components/ApiDataSetChangelog.tsx +++ b/src/explore-education-statistics-common/src/modules/data-catalogue/components/ApiDataSetChangelog.tsx @@ -31,7 +31,7 @@ export default function ApiDataSetChangelog({ this version.

-

Note that the following are considered breaking changes:

+

The following are examples of breaking changes:

  • options were deleted
  • ids were changed
    • diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataSetFilePage.tsx b/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataSetFilePage.tsx index 6b1f07c57f1..e492317ca1b 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataSetFilePage.tsx +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/DataSetFilePage.tsx @@ -259,8 +259,9 @@ export default function DataSetFilePage({ {apiDataSetVersionChanges && ( )} diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/DataSetFileApiChangelog.tsx b/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/DataSetFileApiChangelog.tsx index 68ed496f9c6..a260492b9b7 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/DataSetFileApiChangelog.tsx +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/DataSetFileApiChangelog.tsx @@ -6,10 +6,15 @@ import React from 'react'; interface Props { changes: ApiDataSetVersionChanges; + guidanceNotes: string; version: string; } -export default function DataSetFileApiChangelog({ changes, version }: Props) { +export default function DataSetFileApiChangelog({ + changes, + guidanceNotes, + version, +}: Props) { const { majorChanges, minorChanges } = changes; if (!Object.keys(majorChanges).length && !Object.keys(minorChanges).length) { @@ -21,6 +26,9 @@ export default function DataSetFileApiChangelog({ changes, version }: Props) { heading={pageSections.apiChangelog} id="apiChangelog" > + {guidanceNotes.length && ( +

    {guidanceNotes}

    + )} { - test('renders correctly with major and minor changes', () => { + test('renders correctly with major and minor changes and public guidance notes', () => { render( { ], }, }} + guidanceNotes={'Guidance notes.\nMultiline content.'} version="2.0" />, ); @@ -29,6 +30,10 @@ describe('DataSetFileApiChangelog', () => { screen.getByRole('heading', { name: 'API data set changelog' }), ).toBeInTheDocument(); + expect(screen.getByTestId('public-guidance-notes')).toHaveTextContent( + 'Guidance notes. Multiline content.', + ); + expect( screen.getByRole('heading', { name: 'Major changes for version 2.0' }), ).toBeInTheDocument(); @@ -59,6 +64,7 @@ describe('DataSetFileApiChangelog', () => { }, minorChanges: {}, }} + guidanceNotes="" version="2.0" />, ); @@ -85,6 +91,7 @@ describe('DataSetFileApiChangelog', () => { ], }, }} + guidanceNotes="" version="2.0" />, ); @@ -98,6 +105,27 @@ describe('DataSetFileApiChangelog', () => { ); }); + test('renders correctly with no public data guidance', () => { + render( + , + ); + + expect(screen.queryByTestId('data-guidance-notes')).not.toBeInTheDocument(); + }); + test('does not render if empty changes', () => { render( { majorChanges: {}, minorChanges: {}, }} + guidanceNotes="" version="2.0" />, ); @@ -113,6 +142,10 @@ describe('DataSetFileApiChangelog', () => { screen.queryByRole('heading', { name: 'API data set changelog' }), ).not.toBeInTheDocument(); + expect( + screen.queryByTestId('public-guidance-notes'), + ).not.toBeInTheDocument(); + expect( screen.queryByRole('heading', { name: 'Major changes for version 2.0' }), ).not.toBeInTheDocument(); From 500d96413854223cc0f4344e4f63f2188c11d463 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Thu, 10 Oct 2024 14:27:01 +0100 Subject: [PATCH 49/80] EES-XXXX - amending run_tests.py to detect failed suites from output.xml rather than fail-fast file --- tests/robot-tests/args_and_variables.py | 9 +- tests/robot-tests/reports.py | 100 ++++++++++++------ tests/robot-tests/run_tests.py | 44 ++++---- tests/robot-tests/scripts/create_snapshots.py | 3 +- tests/robot-tests/test_runners.py | 36 ++++--- tests/robot-tests/tests/libs/fail_fast.py | 25 ++--- tests/robot-tests/tests/libs/utilities.py | 22 ++-- 7 files changed, 137 insertions(+), 102 deletions(-) diff --git a/tests/robot-tests/args_and_variables.py b/tests/robot-tests/args_and_variables.py index 6c512bcc64a..b9afdc7a3b6 100644 --- a/tests/robot-tests/args_and_variables.py +++ b/tests/robot-tests/args_and_variables.py @@ -31,7 +31,7 @@ def create_argument_parser() -> argparse.ArgumentParser: dest="processes", type=int, default=4, - help="how many processes should be used when using the pabot interpreter" + help="how many processes should be used when using the pabot interpreter", ) parser.add_argument( "-e", @@ -82,12 +82,7 @@ def create_argument_parser() -> argparse.ArgumentParser: action="store_true", help="rerun failed test suites and merge results into original run results", ) - parser.add_argument( - "--rerun-attempts", - dest="rerun_attempts", - type=int, - default=0, - help="Number of rerun attempts") + parser.add_argument("--rerun-attempts", dest="rerun_attempts", type=int, default=0, help="Number of rerun attempts") parser.add_argument( "--print-keywords", dest="print_keywords", diff --git a/tests/robot-tests/reports.py b/tests/robot-tests/reports.py index 56b832e0c8a..8e39b7cb502 100644 --- a/tests/robot-tests/reports.py +++ b/tests/robot-tests/reports.py @@ -1,7 +1,7 @@ -import os import glob +import os import shutil -import time + from bs4 import BeautifulSoup from robot import rebot_cli as robot_rebot_cli from tests.libs.logger import get_logger @@ -13,21 +13,19 @@ # Merge multiple Robot test reports and assets together into the main test results folder. def merge_robot_reports(first_run_attempt_number: int, number_of_test_runs: int): - - first_run_folder=f"{main_results_folder}{os.sep}run-{first_run_attempt_number}" + first_run_folder = f"{main_results_folder}{os.sep}run-{first_run_attempt_number}" logger.info(f"Merging test run {first_run_attempt_number} results into full results") - + for file in os.listdir(first_run_folder): _copy_to_destination_folder(first_run_folder, file, main_results_folder) for test_run in range(first_run_attempt_number + 1, number_of_test_runs + 1): - logger.info(f"Merging test run {test_run} results into full results") - + test_run_foldername = f"{main_results_folder}{os.sep}run-{test_run}" _merge_test_reports(test_run_foldername) - + for file in glob.glob(rf"{test_run_foldername}{os.sep}*screenshot*"): _copy_to_destination_folder(test_run_foldername, file.split(os.sep)[-1], main_results_folder) @@ -35,11 +33,39 @@ def merge_robot_reports(first_run_attempt_number: int, number_of_test_runs: int) _copy_to_destination_folder(test_run_foldername, file.split(os.sep)[-1], main_results_folder) -def log_report_results(number_of_test_runs: int, failing_suites: []): +def get_failing_test_suite_sources(path_to_report_file: str) -> []: + with open(path_to_report_file, "rb") as report_file: + report_contents = report_file.read() + report = BeautifulSoup(report_contents, features="xml") + + failing_suite_ids = _get_failing_leaf_suite_ids_from_report(report) + suite_elements = report.find_all("suite", recursive=True) + return [ + suite_element.get("source") + for suite_element in suite_elements + if suite_element.get("id") in failing_suite_ids + ] + + +def log_report_results(number_of_test_runs: int, reran_failing_suites: bool, failing_suites: []): + logger.info("*************************************") + logger.info("FINAL REPORT") logger.info(f"Log available at: file://{os.getcwd()}{os.sep}{main_results_folder}{os.sep}log.html") logger.info(f"Report available at: file://{os.getcwd()}{os.sep}{main_results_folder}{os.sep}report.html") - logger.info(f"Number of test runs: {number_of_test_runs}") - + logger.info("*************************************\n") + + if reran_failing_suites: + logger.info("*************************************") + logger.info("LAST RUN ATTEMPT REPORT") + logger.info( + f"Log available at: file://{os.getcwd()}{os.sep}{main_results_folder}{os.sep}run-{number_of_test_runs}{os.sep}log.html" + ) + logger.info( + f"Report available at: file://{os.getcwd()}{os.sep}{main_results_folder}{os.sep}run-{number_of_test_runs}{os.sep}report.html" + ) + logger.info("*************************************\n") + + logger.info(f"Number of test run attempts: {number_of_test_runs}") if failing_suites: logger.info(f"Number of failing suites: {len(failing_suites)}") logger.info(f"Failing suites:") @@ -56,7 +82,7 @@ def create_report_from_output_xml(test_results_folder: str): "output.xml", "--xunit", "xunit.xml", - f"{test_results_folder}/output.xml" + f"{test_results_folder}/output.xml", ] robot_rebot_cli(create_args, exit=False) @@ -79,36 +105,48 @@ def _merge_test_reports(test_results_folder): def filter_out_passing_suites_from_report_file(path_to_original_report: str, path_to_filtered_report: str): - - with open(path_to_original_report, "rb") as report: - - report_contents = report.read() + with open(path_to_original_report, "rb") as report_file: + report_contents = report_file.read() report = BeautifulSoup(report_contents, features="xml") - + passing_suite_ids = _get_passing_suite_ids_from_report(report) - - suite_elements = report.find_all('suite', recursive=True) - [suite_element.extract() for suite_element in suite_elements if suite_element.get('id') in passing_suite_ids] - - suite_stats = report.find_all('stat', recursive=True) - [suite_stat.extract() for suite_stat in suite_stats if suite_stat.get('id') in passing_suite_ids] - + + suite_elements = report.find_all("suite", recursive=True) + [suite_element.extract() for suite_element in suite_elements if suite_element.get("id") in passing_suite_ids] + + suite_stats = report.find_all("stat", recursive=True) + [suite_stat.extract() for suite_stat in suite_stats if suite_stat.get("id") in passing_suite_ids] + if os.path.exists(path_to_filtered_report): os.remove(path_to_filtered_report) with open(path_to_filtered_report, "a") as filtered_file: filtered_file.write(report.prettify()) - -def _get_passing_suite_ids_from_report(report: BeautifulSoup) -> []: - suite_results = report.find('statistics').find('suite').find_all('stat') - passing_suite_results = [suite_result for suite_result in suite_results if int(suite_result.get('fail')) == 0] - return [passing_suite.get('id') for passing_suite in passing_suite_results] + +def _get_passing_suite_ids_from_report(report: BeautifulSoup) -> []: + suite_results = report.find("statistics").find("suite").find_all("stat") + passing_suite_results = [suite_result for suite_result in suite_results if int(suite_result.get("fail")) == 0] + return [passing_suite.get("id") for passing_suite in passing_suite_results] + + +def _get_failing_leaf_suite_ids_from_report(report: BeautifulSoup) -> []: + suite_results = report.find("statistics").find("suite").find_all("stat") + failing_suite_results = [suite_result for suite_result in suite_results if int(suite_result.get("fail")) > 0] + suite_ids = [failing_suite.get("id") for failing_suite in failing_suite_results] + leaf_suite_ids = [] + for suite_id in suite_ids: + other_suite_ids_containing_suite_id = [ + other_suite_id for other_suite_id in suite_ids if other_suite_id != suite_id and suite_id in other_suite_id + ] + if len(other_suite_ids_containing_suite_id) == 0: + leaf_suite_ids += [suite_id] + return leaf_suite_ids def _copy_to_destination_folder(source_folder: str, source_file: str, destination_folder: str): - source_file_path=f"{source_folder}{os.sep}{source_file}" - destination_file_path=f"{destination_folder}{os.sep}{source_file}" + source_file_path = f"{source_folder}{os.sep}{source_file}" + destination_file_path = f"{destination_folder}{os.sep}{source_file}" if os.path.isfile(source_file_path): shutil.copy(source_file_path, destination_file_path) else: diff --git a/tests/robot-tests/run_tests.py b/tests/robot-tests/run_tests.py index 36b82e76cbc..36ce358c3ed 100755 --- a/tests/robot-tests/run_tests.py +++ b/tests/robot-tests/run_tests.py @@ -20,14 +20,14 @@ import admin_api as admin_api import args_and_variables as args_and_variables +import reports +import test_runners import tests.libs.selenium_elements as selenium_elements from scripts.get_webdriver import get_webdriver from tests.libs.create_emulator_release_files import ReleaseFilesGenerator -from tests.libs.fail_fast import failing_suites_filename, get_failing_test_suites +from tests.libs.fail_fast import failing_suites_filename from tests.libs.logger import get_logger from tests.libs.slack import SlackService -import reports -import test_runners pabot_suite_names_filename = ".pabotsuitenames" main_results_folder = "test-results" @@ -87,23 +87,24 @@ def _setup_main_results_folder_for_first_run(args: argparse.Namespace): if args.rerun_failed_suites: previous_report_file_path = f"{main_results_folder}{os.sep}output.xml" if not os.path.exists(previous_report_file_path): - logger.error(f'No previous report file found at {previous_report_file_path} - unable to rerun failed suites') + logger.error( + f"No previous report file found at {previous_report_file_path} - unable to rerun failed suites" + ) sys.exit(1) logger.info(f"Clearing old run folders prior to rerunning failed tests") for old_run_folders in glob.glob(rf"{main_results_folder}{os.sep}run-*"): shutil.rmtree(old_run_folders) - test_run_1_folder = f'{main_results_folder}{os.sep}run-0' - logger.info(f"Copying previous test results into new \"{test_run_1_folder}\" folder") - shutil.copytree(main_results_folder, test_run_1_folder) + test_run_1_folder = f"{main_results_folder}{os.sep}run-0" + logger.info(f'Copying previous test results into new "{test_run_1_folder}" folder') + shutil.copytree(main_results_folder, test_run_1_folder) else: # Remove any existing test results if running from scratch. if Path(main_results_folder).exists(): shutil.rmtree(main_results_folder) - os.mkdir(main_results_folder) + os.mkdir(main_results_folder) def run(): - args = args_and_variables.initialise() _setup_main_results_folder_for_first_run(args) @@ -120,7 +121,7 @@ def run(): _install_chromedriver(args.chromedriver_version) run_identifier_initial_value = _create_run_identifier() - + max_run_attempts = args.rerun_attempts + 1 test_run_index = 0 @@ -136,7 +137,7 @@ def run(): selenium_elements.clear_instances() rerunning_failed_suites = args.rerun_failed_suites or test_run_index > 0 - + # Perform any cleanup before the test run. _clear_files_before_next_test_run_attempt(rerunning_failed_suites) @@ -159,17 +160,21 @@ def run(): # any failed tests from the previous run. if rerunning_failed_suites: previous_report_file = f"{main_results_folder}{os.sep}run-{test_run_index}{os.sep}output.xml" - logger.info(f"Using previous test run's results file \"{previous_report_file}\" to determine which failing suites to run") + logger.info( + f'Using previous test run\'s results file "{previous_report_file}" to determine which failing suites to run' + ) else: previous_report_file = None # Run the tests. - logger.info(f"Performing test run {test_run_index + 1} in test run folder \"{test_run_results_folder}\" with unique identifier {run_identifier}") + logger.info( + f'Performing test run {test_run_index + 1} in test run folder "{test_run_results_folder}" with unique identifier {run_identifier}' + ) test_runners.execute_tests(args, test_run_results_folder, previous_report_file) - + if test_run_index > 0: reports.create_report_from_output_xml(test_run_results_folder) - + finally: # Tear down any data created by this test run unless we've disabled teardown. if args_and_variables.includes_data_changing_tests(args) and not args.disable_teardown: @@ -178,19 +183,20 @@ def run(): test_run_index += 1 + failing_suites = reports.get_failing_test_suite_sources(f"{test_run_results_folder}{os.sep}output.xml") + # If all tests passed, return early. - if not get_failing_test_suites(): + if len(failing_suites) == 0: break - failing_suites = get_failing_test_suites() - # Merge together all reports from all test runs. number_of_test_runs = test_run_index first_run_folder_number = 0 if args.rerun_failed_suites else 1 + reports.merge_robot_reports(first_run_folder_number, number_of_test_runs) # Log the results of the merge test runs. - reports.log_report_results(number_of_test_runs, failing_suites) + reports.log_report_results(number_of_test_runs, rerunning_failed_suites, failing_suites) if args.enable_slack_notifications: slack_service = SlackService() diff --git a/tests/robot-tests/scripts/create_snapshots.py b/tests/robot-tests/scripts/create_snapshots.py index 4aa3cf6b89b..e236c63ac97 100644 --- a/tests/robot-tests/scripts/create_snapshots.py +++ b/tests/robot-tests/scripts/create_snapshots.py @@ -11,8 +11,7 @@ from selenium.common import NoSuchElementException from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.support.ui import Select +from selenium.webdriver.support.ui import Select, WebDriverWait from slack_sdk.webhook import WebhookClient """ diff --git a/tests/robot-tests/test_runners.py b/tests/robot-tests/test_runners.py index b28f58f2680..401af758ba9 100644 --- a/tests/robot-tests/test_runners.py +++ b/tests/robot-tests/test_runners.py @@ -1,7 +1,8 @@ -import os -import reports import argparse +import os + import pabot.pabot as pabot +import reports import robot from tests.libs.logger import get_logger @@ -19,9 +20,9 @@ def create_robot_arguments(arguments: argparse.Namespace, test_run_folder: str) ] robot_args += _create_include_and_exclude_args(arguments) - + robot_args += ["-v", f"timeout:{os.getenv('TIMEOUT')}", "-v", f"implicit_wait:{os.getenv('IMPLICIT_WAIT')}"] - + if arguments.fail_fast: robot_args += ["--exitonfailure"] if arguments.print_keywords: @@ -30,7 +31,7 @@ def create_robot_arguments(arguments: argparse.Namespace, test_run_folder: str) # NOTE(mark): Ensure secrets aren't visible in CI logs/reports robot_args += ["--removekeywords", "name:operatingsystem.environment variable should be set"] robot_args += ["--removekeywords", "name:common.user goes to url"] # To hide basic auth credentials - + if arguments.visual: robot_args += ["-v", "headless:0"] else: @@ -72,7 +73,6 @@ def _create_include_and_exclude_args(arguments: argparse.Namespace) -> []: def execute_tests(arguments: argparse.Namespace, test_run_folder: str, path_to_previous_report_file: str): - robot_args = create_robot_arguments(arguments, test_run_folder) if arguments.interp == "robot": @@ -80,23 +80,25 @@ def execute_tests(arguments: argparse.Namespace, test_run_folder: str, path_to_p robot_args += ["--rerunfailedsuites", path_to_previous_report_file] robot_args += [arguments.tests] - - logger.info(f'Performing test run with Robot') + + logger.info(f"Performing test run with Robot") robot.run_cli(robot_args, exit=False) elif arguments.interp == "pabot": - - robot_args = ['--processes', arguments.processes] + robot_args + robot_args = ["--processes", arguments.processes] + robot_args if path_to_previous_report_file is not None: - - path_to_filtered_report_file = '_filtered.xml'.join(path_to_previous_report_file.rsplit('.xml', 1)) - reports.filter_out_passing_suites_from_report_file(path_to_previous_report_file, path_to_filtered_report_file) - - logger.info(f'Generated filtered report file containing only failing suites at {path_to_filtered_report_file}') - robot_args = ['--suitesfrom', path_to_filtered_report_file] + robot_args + path_to_filtered_report_file = "_filtered.xml".join(path_to_previous_report_file.rsplit(".xml", 1)) + reports.filter_out_passing_suites_from_report_file( + path_to_previous_report_file, path_to_filtered_report_file + ) + + logger.info( + f"Generated filtered report file containing only failing suites at {path_to_filtered_report_file}" + ) + robot_args = ["--suitesfrom", path_to_filtered_report_file] + robot_args robot_args += [arguments.tests] - logger.info(f'Performing test run with Pabot ({arguments.processes} processes)') + logger.info(f"Performing test run with Pabot ({arguments.processes} processes)") pabot.main_program(robot_args) diff --git a/tests/robot-tests/tests/libs/fail_fast.py b/tests/robot-tests/tests/libs/fail_fast.py index c9f075e514f..b3fac71af57 100644 --- a/tests/robot-tests/tests/libs/fail_fast.py +++ b/tests/robot-tests/tests/libs/fail_fast.py @@ -6,28 +6,29 @@ """ import datetime -import os.path import os +import os.path import threading -from robot.libraries.BuiltIn import BuiltIn +import tests.libs.visual as visual from robot.api import SkipExecution +from robot.libraries.BuiltIn import BuiltIn from tests.libs.logger import get_logger from tests.libs.selenium_elements import sl -import tests.libs.visual as visual failing_suites_filename = ".failing_suites" logger = get_logger(__name__) + def record_test_failure(): if not current_test_suite_failing_fast(): record_failing_test_suite() visual.capture_screenshot() visual.capture_large_screenshot() _capture_html() - - if BuiltIn().get_variable_value("${prompt_to_continue_on_failure}") == '1': + + if BuiltIn().get_variable_value("${prompt_to_continue_on_failure}") == "1": _prompt_to_continue() @@ -47,13 +48,13 @@ def current_test_suite_failing_fast() -> bool: def record_failing_test_suite(): if current_test_suite_failing_fast(): return - + test_suite = _get_current_test_suite() - + logger.info( f"Recording test suite '{test_suite}' as failing - subsequent tests will automatically fail in this suite" ) - + with file_lock: try: with open(failing_suites_filename, "a") as file_write: @@ -64,7 +65,7 @@ def record_failing_test_suite(): def fail_test_fast_if_required(): if current_test_suite_failing_fast(): - raise SkipExecution(f"Test suite {_get_current_test_suite()} is already failing. Skipping this test.") + raise SkipExecution("") def get_failing_test_suites() -> []: @@ -82,7 +83,7 @@ def get_failing_test_suites() -> []: def _capture_html(): html = sl().get_source() current_time_millis = round(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) - output_dir = BuiltIn().get_variable_value('${OUTPUT DIR}') + output_dir = BuiltIn().get_variable_value("${OUTPUT DIR}") html_file = open(f"{output_dir}{os.sep}captured-html-{current_time_millis}.html", "w", encoding="utf-8") html_file.write(html) html_file.close() @@ -93,8 +94,8 @@ def _prompt_to_continue(): logger.warn("Continue? (Y/n)") choice = input() if choice.lower().startswith("n"): - raise_assertion_error("Tests stopped!") - + _raise_assertion_error("Tests stopped!") + def _raise_assertion_error(err_msg): sl().failure_occurred() diff --git a/tests/robot-tests/tests/libs/utilities.py b/tests/robot-tests/tests/libs/utilities.py index 11ff15e68cc..ea450e63541 100644 --- a/tests/robot-tests/tests/libs/utilities.py +++ b/tests/robot-tests/tests/libs/utilities.py @@ -1,5 +1,4 @@ import base64 -import datetime import json import os import re @@ -8,7 +7,6 @@ from urllib.parse import urlparse, urlunparse import utilities_init -import visual from robot.libraries.BuiltIn import BuiltIn from selenium.common.exceptions import NoSuchElementException from selenium.webdriver.common.by import By @@ -96,7 +94,7 @@ def retry_or_fail_with_delay(func, retries=5, delay=1.0, *args, **kwargs): def user_waits_until_parent_contains_element( - parent_locator: object, + parent_locator_or_element: object, child_locator: str, timeout: int = None, error: str = None, @@ -105,13 +103,13 @@ def user_waits_until_parent_contains_element( delay: float = 1.0, ): try: - default_timeout = BuiltIn().get_variable_value('${TIMEOUT}') + default_timeout = BuiltIn().get_variable_value("${TIMEOUT}") timeout_per_retry = timeout / retries if timeout is not None else int(default_timeout) / retries child_locator = _normalise_child_locator(child_locator) def parent_contains_matching_element() -> bool: - parent_el = _get_parent_webelement_from_locator(parent_locator, timeout_per_retry, error) + parent_el = _get_parent_webelement_from_locator(parent_locator_or_element, timeout_per_retry, error) return element_finder().find(child_locator, required=False, parent=parent_el) is not None if is_noney(count): @@ -120,7 +118,7 @@ def parent_contains_matching_element() -> bool: retries, delay, parent_contains_matching_element, - "Parent '%s' did not contain '%s' in ." % (parent_locator, child_locator), + "Parent '%s' did not contain '%s' in ." % (parent_locator_or_element, child_locator), timeout_per_retry, error, ) @@ -128,7 +126,7 @@ def parent_contains_matching_element() -> bool: count = int(count) def parent_contains_matching_elements() -> bool: - parent_el = _get_parent_webelement_from_locator(parent_locator, timeout_per_retry, error) + parent_el = _get_parent_webelement_from_locator(parent_locator_or_element, timeout_per_retry, error) return len(sl().find_elements(child_locator, parent=parent_el)) == count retry_or_fail_with_delay( @@ -136,14 +134,15 @@ def parent_contains_matching_elements() -> bool: retries, delay, parent_contains_matching_elements, - "Parent '%s' did not contain %s '%s' element(s) within ." % (parent_locator, count, child_locator), + "Parent '%s' did not contain %s '%s' element(s) within ." + % (parent_locator_or_element, count, child_locator), timeout_per_retry, error, ) except Exception as err: logger.warn( f"Error whilst executing utilities.py user_waits_until_parent_contains_element() " - f"with parent {parent_locator} and child locator {child_locator} - {err}" + f"with parent {parent_locator_or_element} and child locator {child_locator} - {err}" ) raise_assertion_error(err) @@ -285,11 +284,6 @@ def user_should_be_at_top_of_page(): raise_assertion_error(f"Windows position Y is {y} not 0! User should be at the top of the page!") -def capture_large_screenshot_and_prompt_to_continue(): - visual.capture_large_screenshot() - prompt_to_continue() - - def user_gets_row_number_with_heading(heading: str, table_locator: str = "css:table"): elem = get_child_element(table_locator, f'xpath:.//tbody/tr/th[text()="{heading}"]/..') rows = get_child_elements(table_locator, "css:tbody tr") From 38844f02f1954f1566f88e02f671ef8f1261b299 Mon Sep 17 00:00:00 2001 From: dfe-sdt Date: Thu, 10 Oct 2024 15:43:47 +0000 Subject: [PATCH 50/80] chore(tests): update test snapshots --- .../tests/snapshots/data_catalogue_snapshot.json | 3 ++- .../tests/snapshots/find_statistics_snapshot.json | 11 +++++++++-- .../tests/snapshots/methodologies_snapshot.json | 1 + .../tests/snapshots/table_tool_snapshot.json | 1 + 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json b/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json index ac6caf130cf..145f6eae7a0 100644 --- a/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json +++ b/tests/robot-tests/tests/snapshots/data_catalogue_snapshot.json @@ -1,5 +1,5 @@ { - "num_datasets": "949 data sets", + "num_datasets": "954 data sets", "themes": [ { "publications": [ @@ -153,6 +153,7 @@ "Provisional T Level results", "Key stage 4 performance", "Key stage 1 and phonics screening check attainment", + "Phonics screening check attainment", "Key stage 2 attainment", "Key stage 2 attainment: National headlines", "Multi-academy trust performance measures at key stage 2", diff --git a/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json b/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json index 30a4d62c2be..c67a64ec37a 100644 --- a/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json +++ b/tests/robot-tests/tests/snapshots/find_statistics_snapshot.json @@ -30,7 +30,7 @@ { "publication_summary": "Apprenticeship starts, achievements and participation. Includes breakdowns by age, sex, ethnicity, subject, provider, geography etc.", "publication_title": "Apprenticeships", - "published": "12 Sep 2024", + "published": "10 Oct 2024", "release_type": "Accredited official statistics", "theme": "Further education" }, @@ -489,6 +489,13 @@ "release_type": "Official statistics", "theme": "Higher education" }, + { + "publication_summary": "Phonics screening check attainment statistics of pupils in England by pupil characteristics, school characteristics, region and local authority.", + "publication_title": "Phonics screening check attainment", + "published": "10 Oct 2024", + "release_type": "Accredited official statistics", + "theme": "School and college outcomes and performance" + }, { "publication_summary": "Local authority (LA) spending plans for education, children's services and social care.", "publication_title": "Planned LA and school expenditure", @@ -527,7 +534,7 @@ { "publication_summary": "Pupil attendance and absence data including termly national statistics and fortnightly statistics in development derived from DfE\u2019s regular attendance data", "publication_title": "Pupil attendance in schools", - "published": "26 Sep 2024", + "published": "10 Oct 2024", "release_type": "Official statistics in development", "theme": "Pupils and schools" }, diff --git a/tests/robot-tests/tests/snapshots/methodologies_snapshot.json b/tests/robot-tests/tests/snapshots/methodologies_snapshot.json index 7628902949c..2d91d1733db 100644 --- a/tests/robot-tests/tests/snapshots/methodologies_snapshot.json +++ b/tests/robot-tests/tests/snapshots/methodologies_snapshot.json @@ -120,6 +120,7 @@ "Level 2 and 3 attainment age 16 to 25", "Multi-academy trust performance measures (Key stages 2, 4 and 5)", "Multiplication tables check attainment", + "Phonics screening check attainment", "Provisional T Level results" ], "theme_heading": "School and college outcomes and performance" diff --git a/tests/robot-tests/tests/snapshots/table_tool_snapshot.json b/tests/robot-tests/tests/snapshots/table_tool_snapshot.json index fc8be533537..7588be14b09 100644 --- a/tests/robot-tests/tests/snapshots/table_tool_snapshot.json +++ b/tests/robot-tests/tests/snapshots/table_tool_snapshot.json @@ -134,6 +134,7 @@ "Multi-academy trust performance measures at key stage 2", "Multi-academy trust performance measures (Key stages 2, 4 and 5)", "Multiplication tables check attainment", + "Phonics screening check attainment", "Provisional T Level results" ], "theme_heading": "School and college outcomes and performance" From f507258aab32bf080e0d342855b4cf87d6604a15 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 11 Oct 2024 09:39:30 +0100 Subject: [PATCH 51/80] EES-XXXX - adding code comments to describe behaviour of leaf suite ids --- tests/robot-tests/reports.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/robot-tests/reports.py b/tests/robot-tests/reports.py index 8e39b7cb502..39a9e688f29 100644 --- a/tests/robot-tests/reports.py +++ b/tests/robot-tests/reports.py @@ -131,6 +131,12 @@ def _get_passing_suite_ids_from_report(report: BeautifulSoup) -> []: def _get_failing_leaf_suite_ids_from_report(report: BeautifulSoup) -> []: + """Get the ids of failing suites from output.xml, filtering out ids from parent levels in the suite hierarchy. + + Given an example suite folder structure of "Top-level folder:Sub-folder:Leaf suite", the report contains stats + for each level, with ids like "s1", "s1-1" and "s1-1-1". This code returns only the leaf ids, so "s1-1-1" in this + example. + """ suite_results = report.find("statistics").find("suite").find_all("stat") failing_suite_results = [suite_result for suite_result in suite_results if int(suite_result.get("fail")) > 0] suite_ids = [failing_suite.get("id") for failing_suite in failing_suite_results] From 8d5ddbf5451a934c2148477c48daaabe87dd39c3 Mon Sep 17 00:00:00 2001 From: Amy Benson Date: Thu, 10 Oct 2024 11:05:22 +0100 Subject: [PATCH 52/80] EES-5311 EES-5312 accessibility improvements for related content --- .../components/RelatedPagesSection.tsx | 24 ++++++------- .../content/components/ReleaseContent.tsx | 14 ++++---- .../src/prototypes/PrototypeMetadata.tsx | 6 ++-- .../src/prototypes/PrototypePreRelease.tsx | 6 ++-- .../src/prototypes/PrototypeRelease.tsx | 6 ++-- .../src/prototypes/PrototypeReleaseData.tsx | 6 ++-- .../src/prototypes/PrototypeReleaseData2.tsx | 6 ++-- .../prototypes/PrototypeReleaseSummary.tsx | 6 ++-- .../src/prototypes/PrototypeReplaceData.tsx | 6 ++-- .../page-view/PrototypeReleaseContent.tsx | 6 ++-- .../src/components/RelatedAside.tsx | 12 ------- ...module.scss => RelatedContent.module.scss} | 0 .../src/components/RelatedContent.tsx | 15 ++++++++ .../src/components/RelatedInformation.tsx | 14 ++++---- .../PublicationReleasePage.tsx | 36 +++++++++---------- .../__tests__/PublicationReleasePage.test.tsx | 8 ++--- 16 files changed, 84 insertions(+), 87 deletions(-) delete mode 100644 src/explore-education-statistics-common/src/components/RelatedAside.tsx rename src/explore-education-statistics-common/src/components/{RelatedItems.module.scss => RelatedContent.module.scss} (100%) create mode 100644 src/explore-education-statistics-common/src/components/RelatedContent.tsx diff --git a/src/explore-education-statistics-admin/src/pages/release/content/components/RelatedPagesSection.tsx b/src/explore-education-statistics-admin/src/pages/release/content/components/RelatedPagesSection.tsx index baac52c2923..7ef60c17acf 100644 --- a/src/explore-education-statistics-admin/src/pages/release/content/components/RelatedPagesSection.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/content/components/RelatedPagesSection.tsx @@ -44,20 +44,18 @@ export default function RelatedPagesSection({ release }: Props) { <> {(editingMode === 'edit' || links.length > 0) && ( <> - - + +
      + {links.map(({ id, description, url }) => ( +
    • + removeLink(id)} to={url}> + {description} + +
      • + ))} +
    )} {editingMode === 'edit' && ( diff --git a/src/explore-education-statistics-admin/src/pages/release/content/components/ReleaseContent.tsx b/src/explore-education-statistics-admin/src/pages/release/content/components/ReleaseContent.tsx index 294f79f34e5..9c9ac61ba5e 100644 --- a/src/explore-education-statistics-admin/src/pages/release/content/components/ReleaseContent.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/content/components/ReleaseContent.tsx @@ -24,7 +24,7 @@ import Button from '@common/components/Button'; import ButtonText from '@common/components/ButtonText'; import Details from '@common/components/Details'; import PageSearchForm from '@common/components/PageSearchForm'; -import RelatedAside from '@common/components/RelatedAside'; +import RelatedContent from '@common/components/RelatedContent'; import ScrollableContainer from '@common/components/ScrollableContainer'; import Tag from '@common/components/Tag'; import ReleaseSummarySection from '@common/modules/release/components/ReleaseSummarySection'; @@ -245,7 +245,7 @@ const ReleaseContent = ({
    - + @@ -335,9 +335,9 @@ const ReleaseContent = ({ {!!releaseSeries.length && ( <> -

    +

    Releases in this series -

    +
    0 && ( <> -

    Methodologies -

    +
      {allMethodologies.map(methodology => (
    • @@ -396,7 +396,7 @@ const ReleaseContent = ({ )} - +
    diff --git a/src/explore-education-statistics-admin/src/prototypes/PrototypeMetadata.tsx b/src/explore-education-statistics-admin/src/prototypes/PrototypeMetadata.tsx index ca4a5a91842..9d7a55167f0 100644 --- a/src/explore-education-statistics-admin/src/prototypes/PrototypeMetadata.tsx +++ b/src/explore-education-statistics-admin/src/prototypes/PrototypeMetadata.tsx @@ -2,7 +2,7 @@ import PageTitle from '@admin/components/PageTitle'; import PrototypePage from '@admin/prototypes/components/PrototypePage'; import React from 'react'; import Tabs from '@common/components/Tabs'; -import RelatedAside from '@common/components/RelatedAside'; +import RelatedContent from '@common/components/RelatedContent'; import TabsSection from '@common/components/TabsSection'; import NavBar from './components/PrototypeNavBar'; import CreateMeta from './components/PrototypeMetaCreate'; @@ -16,7 +16,7 @@ const PrototypeMetadata = () => {
    - +

    Help and guidance

    • @@ -25,7 +25,7 @@ const PrototypeMetadata = () => {
    -     +
    diff --git a/src/explore-education-statistics-admin/src/prototypes/PrototypePreRelease.tsx b/src/explore-education-statistics-admin/src/prototypes/PrototypePreRelease.tsx index d72d52345fb..c3ce10b59be 100644 --- a/src/explore-education-statistics-admin/src/prototypes/PrototypePreRelease.tsx +++ b/src/explore-education-statistics-admin/src/prototypes/PrototypePreRelease.tsx @@ -2,7 +2,7 @@ import PageTitle from '@admin/components/PageTitle'; import PrototypePage from '@admin/prototypes/components/PrototypePage'; import React from 'react'; import Tabs from '@common/components/Tabs'; -import RelatedAside from '@common/components/RelatedAside'; +import RelatedContent from '@common/components/RelatedContent'; import TabsSection from '@common/components/TabsSection'; import NavBar from './components/PrototypeNavBar'; import CreatePreRelease from './components/PrototypePreReleaseCreate'; @@ -16,7 +16,7 @@ const PrototypePreReleasePage = () => {
    - +

    Help and guidance

    • @@ -25,7 +25,7 @@ const PrototypePreReleasePage = () => {
    -     +
    diff --git a/src/explore-education-statistics-admin/src/prototypes/PrototypeRelease.tsx b/src/explore-education-statistics-admin/src/prototypes/PrototypeRelease.tsx index 657dc7209d3..4fa0d2bcf9e 100644 --- a/src/explore-education-statistics-admin/src/prototypes/PrototypeRelease.tsx +++ b/src/explore-education-statistics-admin/src/prototypes/PrototypeRelease.tsx @@ -4,7 +4,7 @@ import Accordion from '@common/components/Accordion'; import AccordionSection from '@common/components/AccordionSection'; import Details from '@common/components/Details'; import PageSearchForm from '@common/components/PageSearchForm'; -import RelatedAside from '@common/components/RelatedAside'; +import RelatedContent from '@common/components/RelatedContent'; import Tabs from '@common/components/Tabs'; import TabsSection from '@common/components/TabsSection'; import stylesKeyStat from '@common/modules/find-statistics/components/KeyStat.module.scss'; @@ -121,7 +121,7 @@ const PrototypeRelease = () => {
    - +

    Related infomation

    -     +
    diff --git a/src/explore-education-statistics-admin/src/prototypes/PrototypeReleaseData.tsx b/src/explore-education-statistics-admin/src/prototypes/PrototypeReleaseData.tsx index bfc118c942c..54dc73a5202 100644 --- a/src/explore-education-statistics-admin/src/prototypes/PrototypeReleaseData.tsx +++ b/src/explore-education-statistics-admin/src/prototypes/PrototypeReleaseData.tsx @@ -3,7 +3,7 @@ import Link from '@admin/components/Link'; import PrototypePage from '@admin/prototypes/components/PrototypePage'; import Details from '@common/components/Details'; import PageSearchForm from '@common/components/PageSearchForm'; -import RelatedAside from '@common/components/RelatedAside'; +import RelatedContent from '@common/components/RelatedContent'; import Tabs from '@common/components/Tabs'; import TabsSection from '@common/components/TabsSection'; import stylesKeyStat from '@common/modules/find-statistics/components/KeyStat.module.scss'; @@ -144,7 +144,7 @@ const PrototypeReleaseData = () => {
    - +

    Quick links

    • @@ -203,7 +203,7 @@ const PrototypeReleaseData = () => {
    -     +
    diff --git a/src/explore-education-statistics-admin/src/prototypes/PrototypeReleaseData2.tsx b/src/explore-education-statistics-admin/src/prototypes/PrototypeReleaseData2.tsx index 5fc6bea8ae8..a6078ac7914 100644 --- a/src/explore-education-statistics-admin/src/prototypes/PrototypeReleaseData2.tsx +++ b/src/explore-education-statistics-admin/src/prototypes/PrototypeReleaseData2.tsx @@ -5,7 +5,7 @@ import Accordion from '@common/components/Accordion'; import AccordionSection from '@common/components/AccordionSection'; import Details from '@common/components/Details'; import PageSearchForm from '@common/components/PageSearchForm'; -import RelatedAside from '@common/components/RelatedAside'; +import RelatedContent from '@common/components/RelatedContent'; import Tabs from '@common/components/Tabs'; import TabsSection from '@common/components/TabsSection'; import stylesKeyStat from '@common/modules/find-statistics/components/KeyStat.module.scss'; @@ -132,7 +132,7 @@ const PrototypeReleaseData = () => {
    - +

    Data downloads

    -     +

    Key statistics and data downloads

    diff --git a/src/explore-education-statistics-admin/src/prototypes/PrototypeReleaseSummary.tsx b/src/explore-education-statistics-admin/src/prototypes/PrototypeReleaseSummary.tsx index bab1548efd3..e32330eca12 100644 --- a/src/explore-education-statistics-admin/src/prototypes/PrototypeReleaseSummary.tsx +++ b/src/explore-education-statistics-admin/src/prototypes/PrototypeReleaseSummary.tsx @@ -1,7 +1,7 @@ import PageTitle from '@admin/components/PageTitle'; import PrototypePage from '@admin/prototypes/components/PrototypePage'; import React from 'react'; -import RelatedAside from '@common/components/RelatedAside'; +import RelatedContent from '@common/components/RelatedContent'; import Button from '@common/components/Button'; import SummaryList from '@common/components/SummaryList'; import SummaryListItem from '@common/components/SummaryListItem'; @@ -33,7 +33,7 @@ const PrototypeReleaseSummary = () => { />
    - +

    Help and guidance

    • @@ -42,7 +42,7 @@ const PrototypeReleaseSummary = () => {
    -     +
    diff --git a/src/explore-education-statistics-admin/src/prototypes/PrototypeReplaceData.tsx b/src/explore-education-statistics-admin/src/prototypes/PrototypeReplaceData.tsx index ca4a5a91842..9d7a55167f0 100644 --- a/src/explore-education-statistics-admin/src/prototypes/PrototypeReplaceData.tsx +++ b/src/explore-education-statistics-admin/src/prototypes/PrototypeReplaceData.tsx @@ -2,7 +2,7 @@ import PageTitle from '@admin/components/PageTitle'; import PrototypePage from '@admin/prototypes/components/PrototypePage'; import React from 'react'; import Tabs from '@common/components/Tabs'; -import RelatedAside from '@common/components/RelatedAside'; +import RelatedContent from '@common/components/RelatedContent'; import TabsSection from '@common/components/TabsSection'; import NavBar from './components/PrototypeNavBar'; import CreateMeta from './components/PrototypeMetaCreate'; @@ -16,7 +16,7 @@ const PrototypeMetadata = () => {
    - +

    Help and guidance

    • @@ -25,7 +25,7 @@ const PrototypeMetadata = () => {
    -     +
    diff --git a/src/explore-education-statistics-admin/src/prototypes/page-view/PrototypeReleaseContent.tsx b/src/explore-education-statistics-admin/src/prototypes/page-view/PrototypeReleaseContent.tsx index 0f8055cfded..85a30a7316f 100644 --- a/src/explore-education-statistics-admin/src/prototypes/page-view/PrototypeReleaseContent.tsx +++ b/src/explore-education-statistics-admin/src/prototypes/page-view/PrototypeReleaseContent.tsx @@ -22,7 +22,7 @@ import Button from '@common/components/Button'; import ButtonText from '@common/components/ButtonText'; import Details from '@common/components/Details'; import PageSearchForm from '@common/components/PageSearchForm'; -import RelatedAside from '@common/components/RelatedAside'; +import RelatedContent from '@common/components/RelatedContent'; import ScrollableContainer from '@common/components/ScrollableContainer'; import Tag from '@common/components/Tag'; import ReleaseSummarySection from '@common/modules/release/components/ReleaseSummarySection'; @@ -207,7 +207,7 @@ const PrototypeReleaseContent = ({
    - + @@ -358,7 +358,7 @@ const PrototypeReleaseContent = ({ )} - +
    diff --git a/src/explore-education-statistics-common/src/components/RelatedAside.tsx b/src/explore-education-statistics-common/src/components/RelatedAside.tsx deleted file mode 100644 index b2845fad08f..00000000000 --- a/src/explore-education-statistics-common/src/components/RelatedAside.tsx +++ /dev/null @@ -1,12 +0,0 @@ -import React, { ReactNode } from 'react'; -import styles from './RelatedItems.module.scss'; - -interface Props { - children: ReactNode; -} - -const RelatedAside = ({ children }: Props) => { - return ; -}; - -export default RelatedAside; diff --git a/src/explore-education-statistics-common/src/components/RelatedItems.module.scss b/src/explore-education-statistics-common/src/components/RelatedContent.module.scss similarity index 100% rename from src/explore-education-statistics-common/src/components/RelatedItems.module.scss rename to src/explore-education-statistics-common/src/components/RelatedContent.module.scss diff --git a/src/explore-education-statistics-common/src/components/RelatedContent.tsx b/src/explore-education-statistics-common/src/components/RelatedContent.tsx new file mode 100644 index 00000000000..58a33bf3bce --- /dev/null +++ b/src/explore-education-statistics-common/src/components/RelatedContent.tsx @@ -0,0 +1,15 @@ +import styles from '@common/components/RelatedContent.module.scss'; +import React, { ReactNode } from 'react'; + +interface Props { + children: ReactNode; + testId?: string; +} + +export default function RelatedContent({ children, testId }: Props) { + return ( +
    + {children} +
    + ); +} diff --git a/src/explore-education-statistics-common/src/components/RelatedInformation.tsx b/src/explore-education-statistics-common/src/components/RelatedInformation.tsx index 8297af23d41..5532afc0ed3 100644 --- a/src/explore-education-statistics-common/src/components/RelatedInformation.tsx +++ b/src/explore-education-statistics-common/src/components/RelatedInformation.tsx @@ -1,4 +1,4 @@ -import RelatedAside from '@common/components/RelatedAside'; +import RelatedContent from '@common/components/RelatedContent'; import React, { ReactNode } from 'react'; interface Props { @@ -7,21 +7,19 @@ interface Props { id?: string; } -const RelatedInformation = ({ +export default function RelatedInformation({ children, heading = 'Related information', id = 'related-information', -}: Props) => { +}: Props) { return ( - + - + ); -}; - -export default RelatedInformation; +} diff --git a/src/explore-education-statistics-frontend/src/modules/find-statistics/PublicationReleasePage.tsx b/src/explore-education-statistics-frontend/src/modules/find-statistics/PublicationReleasePage.tsx index 26d88d81062..0e592437c96 100644 --- a/src/explore-education-statistics-frontend/src/modules/find-statistics/PublicationReleasePage.tsx +++ b/src/explore-education-statistics-frontend/src/modules/find-statistics/PublicationReleasePage.tsx @@ -2,7 +2,7 @@ import Accordion from '@common/components/Accordion'; import AccordionSection from '@common/components/AccordionSection'; import Details from '@common/components/Details'; import FormattedDate from '@common/components/FormattedDate'; -import RelatedAside from '@common/components/RelatedAside'; +import RelatedContent from '@common/components/RelatedContent'; import Tag from '@common/components/Tag'; import VisuallyHidden from '@common/components/VisuallyHidden'; import ScrollableContainer from '@common/components/ScrollableContainer'; @@ -207,7 +207,7 @@ const PublicationReleasePage: NextPage = ({ release }) => {
    - + @@ -320,9 +320,9 @@ const PublicationReleasePage: NextPage = ({ release }) => {
{!!releaseSeries.length && ( <> -

+

Releases in this series -

+
= ({ release }) => { {(release.publication.methodologies.length > 0 || release.publication.externalMethodology) && ( <> -

Methodologies -

+
    {release.publication.methodologies.map(methodology => (
  • @@ -401,22 +401,20 @@ const PublicationReleasePage: NextPage = ({ release }) => { )} {release.relatedInformation.length > 0 && ( <> - - + +
      + {release.relatedInformation && + release.relatedInformation.map(link => ( +
    • + {link.description} +
      • + ))} +
    )} - +
    diff --git a/src/explore-education-statistics-frontend/src/modules/find-statistics/__tests__/PublicationReleasePage.test.tsx b/src/explore-education-statistics-frontend/src/modules/find-statistics/__tests__/PublicationReleasePage.test.tsx index 5a1d9ef1467..4dbfc470915 100644 --- a/src/explore-education-statistics-frontend/src/modules/find-statistics/__tests__/PublicationReleasePage.test.tsx +++ b/src/explore-education-statistics-frontend/src/modules/find-statistics/__tests__/PublicationReleasePage.test.tsx @@ -190,7 +190,7 @@ describe('PublicationReleasePage', () => { test(`renders other releases including legacy links`, async () => { render(); - const usefulInfo = within(screen.getByRole('complementary')); + const usefulInfo = within(screen.getByTestId('useful-information')); expect( usefulInfo.getByRole('heading', { name: 'Releases in this series' }), @@ -393,7 +393,7 @@ describe('PublicationReleasePage', () => { test('renders link to a methodology', () => { render(); - const usefulInfo = screen.getByRole('complementary'); + const usefulInfo = screen.getByTestId('useful-information'); expect( within(usefulInfo).getByRole('heading', { name: 'Methodologies' }), @@ -424,7 +424,7 @@ describe('PublicationReleasePage', () => { render( , ); - const usefulInfo = screen.getByRole('complementary'); + const usefulInfo = screen.getByTestId('useful-information'); expect( within(usefulInfo).getByRole('heading', { name: 'Methodologies' }), @@ -453,7 +453,7 @@ describe('PublicationReleasePage', () => { release={testReleaseWithInternalAndExternalMethodologies} />, ); - const usefulInfo = screen.getByRole('complementary'); + const usefulInfo = screen.getByTestId('useful-information'); expect( within(usefulInfo).getByRole('heading', { name: 'Methodologies' }), From cfde347323448445de66e88cc3c4ed3085fc42ad Mon Sep 17 00:00:00 2001 From: Amy Benson Date: Thu, 10 Oct 2024 16:51:57 +0100 Subject: [PATCH 53/80] EES-5297 add visually hidden text to map control labels --- .../src/modules/charts/components/MapBlock.tsx | 2 ++ .../modules/charts/components/MapControls.tsx | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/explore-education-statistics-common/src/modules/charts/components/MapBlock.tsx b/src/explore-education-statistics-common/src/modules/charts/components/MapBlock.tsx index 4ff06d7dbd3..c35985fa1fe 100644 --- a/src/explore-education-statistics-common/src/modules/charts/components/MapBlock.tsx +++ b/src/explore-education-statistics-common/src/modules/charts/components/MapBlock.tsx @@ -120,6 +120,7 @@ export default function MapBlock({ width, height, axes, + title, }: MapBlockProps) { const axisMajor = useMemo( () => ({ @@ -221,6 +222,7 @@ export default function MapBlock({ id={id} selectedDataSetKey={selectedDataSetKey} selectedLocation={selectedFeature?.id?.toString()} + title={title} onChangeDataSet={setSelectedDataSetKey} onChangeLocation={value => { const feature = features?.features.find(feat => feat.id === value); diff --git a/src/explore-education-statistics-common/src/modules/charts/components/MapControls.tsx b/src/explore-education-statistics-common/src/modules/charts/components/MapControls.tsx index e035f0f657a..c63d77cffd2 100644 --- a/src/explore-education-statistics-common/src/modules/charts/components/MapControls.tsx +++ b/src/explore-education-statistics-common/src/modules/charts/components/MapControls.tsx @@ -1,5 +1,6 @@ import { FormGroup, FormSelect } from '@common/components/form'; import { SelectOption } from '@common/components/form/FormSelect'; +import VisuallyHidden from '@common/components/VisuallyHidden'; import { MapDataSetCategory } from '@common/modules/charts/components/utils/createMapDataSetCategories'; import { Dictionary } from '@common/types'; import locationLevelsMap, { @@ -15,6 +16,7 @@ interface Props { id: string; selectedDataSetKey: string; selectedLocation?: string; + title?: string; onChangeDataSet: (value: string) => void; onChangeLocation: (value: string) => void; } @@ -25,6 +27,7 @@ export default function MapControls({ id, selectedDataSetKey, selectedLocation, + title, onChangeDataSet, onChangeLocation, }: Props) { @@ -92,7 +95,12 @@ export default function MapControls({ name="selectedDataSet" id={`${id}-selectedDataSet`} className="govuk-!-width-full" - label="1. Select data to view" + label={ + <> + 1. Select data to view + {title && {` for ${title}`}} + + } value={selectedDataSetKey} onChange={e => onChangeDataSet(e.currentTarget.value)} options={dataSetOptions} @@ -104,7 +112,12 @@ export default function MapControls({ + {`2. Select ${locationType.prefix} ${locationType.label}`} + {title && {` for ${title}`}} + + } value={selectedLocation} options={locationOptions} optGroups={groupedLocationOptions} From f08ac1cf897b67257ac6b49e66224f6ef4c6f271 Mon Sep 17 00:00:00 2001 From: Amy Benson Date: Thu, 10 Oct 2024 09:51:21 +0100 Subject: [PATCH 54/80] EES-5434 fix footnotes heading level on permalinks --- .../src/components/FigureFootnotes.tsx | 29 ++++++++++++++----- .../components/FixedMultiHeaderDataTable.tsx | 3 ++ .../src/modules/permalink/PermalinkPage.tsx | 1 + 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/explore-education-statistics-common/src/components/FigureFootnotes.tsx b/src/explore-education-statistics-common/src/components/FigureFootnotes.tsx index 2fb234ccd86..f060e5ce9a9 100644 --- a/src/explore-education-statistics-common/src/components/FigureFootnotes.tsx +++ b/src/explore-education-statistics-common/src/components/FigureFootnotes.tsx @@ -2,23 +2,36 @@ import CollapsibleList from '@common/components/CollapsibleList'; import ContentHtml from '@common/components/ContentHtml'; import VisuallyHidden from '@common/components/VisuallyHidden'; import { Footnote } from '@common/services/types/footnotes'; -import React from 'react'; +import React, { createElement } from 'react'; interface Props { footnotes: Footnote[]; headingHiddenText?: string; + headingTag?: 'h2' | 'h3'; id: string; } -const FigureFootnotes = ({ footnotes, headingHiddenText, id }: Props) => { +const FigureFootnotes = ({ + footnotes, + headingHiddenText, + headingTag = 'h3', + id, +}: Props) => { return footnotes.length > 0 ? ( <> -

    - Footnotes - {headingHiddenText && ( - {` ${headingHiddenText}`} - )} -

    + {createElement( + headingTag, + { + className: `govuk-heading-m`, + }, + <> + Footnotes + {headingHiddenText && ( + {` ${headingHiddenText}`} + )} + , + )} + { innerRef?: Ref; footnotes?: FullTableMeta['footnotes']; footnotesClassName?: string; + footnotesHeadingTag?: 'h2' | 'h3'; footnotesId: string; source?: string; footnotesHeadingHiddenText?: string; @@ -25,6 +26,7 @@ const FixedMultiHeaderDataTable = forwardRef( captionId, footnotes = [], footnotesClassName, + footnotesHeadingTag, footnotesId, source, footnotesHeadingHiddenText, @@ -116,6 +118,7 @@ const FixedMultiHeaderDataTable = forwardRef( diff --git a/src/explore-education-statistics-frontend/src/modules/permalink/PermalinkPage.tsx b/src/explore-education-statistics-frontend/src/modules/permalink/PermalinkPage.tsx index 9f05016a9c2..ae2d7f6a5c6 100644 --- a/src/explore-education-statistics-frontend/src/modules/permalink/PermalinkPage.tsx +++ b/src/explore-education-statistics-frontend/src/modules/permalink/PermalinkPage.tsx @@ -87,6 +87,7 @@ const PermalinkPage: NextPage = ({ data }) => { captionId={captionId} footnotes={footnotes} footnotesClassName="govuk-!-width-two-thirds" + footnotesHeadingTag="h2" footnotesId={footnotesId} source={`${publicationTitle}, ${dataSetTitle}`} tableJson={json} From 86af70f89a1359d1be472deb63f009bd4828f5af Mon Sep 17 00:00:00 2001 From: Amy Benson Date: Thu, 10 Oct 2024 14:53:45 +0100 Subject: [PATCH 55/80] EES-5284 add border to map legend colours --- .../src/modules/charts/components/MapBlock.module.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/src/explore-education-statistics-common/src/modules/charts/components/MapBlock.module.scss b/src/explore-education-statistics-common/src/modules/charts/components/MapBlock.module.scss index fe185f2eb00..5b437bcf2b6 100644 --- a/src/explore-education-statistics-common/src/modules/charts/components/MapBlock.module.scss +++ b/src/explore-education-statistics-common/src/modules/charts/components/MapBlock.module.scss @@ -26,6 +26,7 @@ } .legendIcon { + border: 1px solid govuk-colour('black'); forced-color-adjust: none; height: 20px; margin-right: govuk-spacing(1); From 35847f3e7b73a4ec5b216c3fd6c6e97c12a90215 Mon Sep 17 00:00:00 2001 From: Mark Youngman Date: Fri, 11 Oct 2024 15:51:07 +0100 Subject: [PATCH 56/80] EES-5565 Fix UI test failure due to existence of other prerelease users --- .../PrereleaseUsersController.cs | 11 +---- .../Services/UserManagementService.cs | 1 - tests/robot-tests/tests/libs/admin_api.py | 48 +++++++++++-------- 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/UserManagement/PrereleaseUsersController.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/UserManagement/PrereleaseUsersController.cs index 452f9b6deb0..7a1370a692d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/UserManagement/PrereleaseUsersController.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/UserManagement/PrereleaseUsersController.cs @@ -25,14 +25,7 @@ public PrereleaseUsersController(IUserManagementService userManagementService) [ProducesResponseType(404)] public async Task>> GetPreReleaseUserList() { - var users = await _userManagementService.ListPreReleaseUsersAsync(); - - if (users.Any()) - { - return Ok(users); - } - - return NotFound(); + return await _userManagementService.ListPreReleaseUsersAsync(); } } -} \ No newline at end of file +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/UserManagementService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/UserManagementService.cs index a66d17bf3a6..6e4f6bd09b4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/UserManagementService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/UserManagementService.cs @@ -5,7 +5,6 @@ using System.Threading.Tasks; using GovUk.Education.ExploreEducationStatistics.Admin.Database; using GovUk.Education.ExploreEducationStatistics.Admin.Models; -using GovUk.Education.ExploreEducationStatistics.Admin.Options; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Admin.Services.Interfaces.Security; using GovUk.Education.ExploreEducationStatistics.Admin.ViewModels; diff --git a/tests/robot-tests/tests/libs/admin_api.py b/tests/robot-tests/tests/libs/admin_api.py index 72a34ad5ee9..239bd0151f3 100644 --- a/tests/robot-tests/tests/libs/admin_api.py +++ b/tests/robot-tests/tests/libs/admin_api.py @@ -2,7 +2,6 @@ from datetime import datetime import requests -from robot.libraries.BuiltIn import BuiltIn from tests.libs import local_storage_helper # To prevent InsecureRequestWarning @@ -166,20 +165,17 @@ def user_resets_user_roles_via_api_if_required(user_emails: list) -> None: for user_email in user_emails: if user_email not in allowed_users: - raise AssertionError(f"`User emails` must contain only allowed users: {allowed_users}") - try: - user_ids = [_get_prerelease_user_details_via_api(user_email)["id"]] - _ = [user_removes_all_release_and_publication_roles_from_user(user_id) for user_id in user_ids] - BuiltIn().log(f"All userReleaseRoles & userPublicationRoles reset for user: {user_email}") - except TypeError or IndexError: - BuiltIn().log(f"User with email {user_email} does not exist in pre-release user list", "WARN") - - try: - user_ids = [_get_user_details_via_api(user_email)["id"]] - _ = [user_removes_all_release_and_publication_roles_from_user(user_id) for user_id in user_ids] - BuiltIn().log(f"All userReleaseRoles & userPublicationRoles reset for user: {user_email}") - except TypeError or IndexError: - BuiltIn().log(f"User with email {user_email} does not exist in user list", "WARN") + raise AssertionError( + f"Not allowed to reset roles for {user_email}. Can only reset user roles for following users: {allowed_users}" + ) + + user = _get_user_details_via_api(user_email) + + if not user: + user = _get_prerelease_user_details_via_api(user_email) + assert user, f"Failed to find user with email {user_email}" + + user_removes_all_release_and_publication_roles_from_user(user["id"]) def user_creates_test_release_via_api( @@ -219,21 +215,35 @@ def delete_test_user(email: str): def _get_user_details_via_api(user_email: str): - users = admin_client.get("/api/user-management/users").json() + response = admin_client.get("/api/user-management/users") + + assert response.status_code == 200, "Error when fetching users via api" + + users = response.json() matching_users = list(filter(lambda user: user["email"] == user_email, users)) - assert matching_users, f"Could not find user with email {user_email}" + if len(matching_users) == 0: + return None + + assert len(matching_users) == 1, f"Should only have found one user with email {user_email}" return matching_users[0] def _get_prerelease_user_details_via_api(user_email: str): - users = admin_client.get("/api/user-management/pre-release").json() + response = admin_client.get("/api/user-management/pre-release") + + assert response.status_code == 200, "Error when fetching prerelease users via api" + + users = response.json() matching_users = list(filter(lambda user: user["email"] == user_email, users)) - assert matching_users, f"Could not find user with email {user_email}" + if len(matching_users) == 0: + return None + + assert len(matching_users) == 1, f"Should only have found one prerelease user with email {user_email}" return matching_users[0] From 63ca029691d6ba41cc30dc523eb2ddb32ddf4766 Mon Sep 17 00:00:00 2001 From: "rian.thwaite" Date: Mon, 14 Oct 2024 09:08:22 +0100 Subject: [PATCH 57/80] EES-5527 remove unucessary hook usage as per PR comments --- .../content/components/EditableKeyStatDataBlock.tsx | 8 +++----- .../release/content/components/EditableKeyStatText.tsx | 8 +++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatDataBlock.tsx b/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatDataBlock.tsx index fe2dc56baf8..80312b7e4df 100644 --- a/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatDataBlock.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatDataBlock.tsx @@ -10,7 +10,7 @@ import useToggle from '@common/hooks/useToggle'; import tableBuilderQueries from '@common/modules/find-statistics/queries/tableBuilderQueries'; import { KeyStatisticDataBlock } from '@common/services/publicationService'; import { useQuery } from '@tanstack/react-query'; -import React, { useCallback, useState } from 'react'; +import React, { useCallback } from 'react'; export interface EditableKeyStatDataBlockProps { isEditing?: boolean; @@ -33,7 +33,6 @@ export default function EditableKeyStatDataBlock({ onRemove, onSubmit, }: EditableKeyStatDataBlockProps) { - const [keyStatisticId, setKeyStatisticId] = useState(''); const [showForm, toggleShowForm] = useToggle(false); const { @@ -47,10 +46,9 @@ export default function EditableKeyStatDataBlock({ const handleSubmit = useCallback( async (values: KeyStatDataBlockFormValues) => { await onSubmit(values); - setKeyStatisticId(''); toggleShowForm.off(); }, - [onSubmit, toggleShowForm, setKeyStatisticId], + [onSubmit, toggleShowForm], ); if (isLoading) { @@ -79,7 +77,7 @@ export default function EditableKeyStatDataBlock({ keyStatTitle === keyStatisticId, + keyStatTitle => keyStatTitle === keyStat.guidanceTitle )} title={title} statistic={statistic} diff --git a/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatText.tsx b/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatText.tsx index b4aadedddc5..f0719fd5a44 100644 --- a/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatText.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatText.tsx @@ -4,7 +4,7 @@ import EditableKeyStatTextForm, { } from '@admin/pages/release/content/components/EditableKeyStatTextForm'; import useToggle from '@common/hooks/useToggle'; import { KeyStatisticText } from '@common/services/publicationService'; -import React, { useCallback, useState } from 'react'; +import React, { useCallback } from 'react'; export interface EditableKeyStatTextProps { isEditing?: boolean; @@ -25,16 +25,14 @@ export default function EditableKeyStatText({ onRemove, onSubmit, }: EditableKeyStatTextProps) { - const [keyStatisticId, setKeyStatisticId] = useState(''); const [showForm, toggleShowForm] = useToggle(false); const handleSubmit = useCallback( async (values: KeyStatTextFormValues) => { await onSubmit(values); - setKeyStatisticId(''); toggleShowForm.off(); }, - [onSubmit, setKeyStatisticId, toggleShowForm], + [onSubmit, toggleShowForm], ); if (showForm) { @@ -42,7 +40,7 @@ export default function EditableKeyStatText({ keyStatTitle === keyStatisticId, + keyStatTitle => keyStatTitle === keyStat.title )} isReordering={isReordering} testId={testId} From 9027c126747e0cce0e095aae0cfa50e1cabbdc76 Mon Sep 17 00:00:00 2001 From: "rian.thwaite" Date: Mon, 14 Oct 2024 10:25:59 +0100 Subject: [PATCH 58/80] EES-5527 fix formatting --- .../release/content/components/EditableKeyStatDataBlock.tsx | 2 +- .../pages/release/content/components/EditableKeyStatText.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatDataBlock.tsx b/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatDataBlock.tsx index 80312b7e4df..1550f53309e 100644 --- a/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatDataBlock.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatDataBlock.tsx @@ -77,7 +77,7 @@ export default function EditableKeyStatDataBlock({ keyStatTitle === keyStat.guidanceTitle + keyStatTitle => keyStatTitle === keyStat.guidanceTitle, )} title={title} statistic={statistic} diff --git a/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatText.tsx b/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatText.tsx index f0719fd5a44..855f5344ee7 100644 --- a/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatText.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/content/components/EditableKeyStatText.tsx @@ -40,7 +40,7 @@ export default function EditableKeyStatText({ keyStatTitle === keyStat.title + keyStatTitle => keyStatTitle === keyStat.title, )} isReordering={isReordering} testId={testId} From 2c81f1e8991bca5ed375a48c570fa46748f1fe83 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Fri, 11 Oct 2024 23:02:26 +0100 Subject: [PATCH 59/80] EES-5570 Cleanup `DataSetVersionMappingControllerTests` This primarily refactors the tests to use a common `CreateInitialAndNextDataSetVersion` method to cut down on test data boilerplate. --- .../DataSetVersionMappingControllerTests.cs | 779 ++++-------------- 1 file changed, 161 insertions(+), 618 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetVersionMappingControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetVersionMappingControllerTests.cs index 14cd869a080..bfdcde74470 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetVersionMappingControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetVersionMappingControllerTests.cs @@ -34,35 +34,11 @@ public class GetLocationMappingsTests( [Fact] public async Task Success() { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion currentDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 0) - .WithStatusPublished() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); - - DataSetVersion nextDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 1) - .WithStatusDraft() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); - await TestApp.AddTestData(context => - { - context.DataSetVersions.AddRange(currentDataSetVersion, nextDataSetVersion); - context.DataSets.Update(dataSet); - }); - - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() - .WithSourceDataSetVersionId(currentDataSetVersion.Id) + .WithSourceDataSetVersionId(initialDataSetVersion.Id) .WithTargetDataSetVersionId(nextDataSetVersion.Id) .WithLocationMappingPlan(DataFixture .DefaultLocationMappingPlan() @@ -107,10 +83,7 @@ await TestApp.AddTestData(context => targetKey: "target-location-1-key", candidate: DataFixture.DefaultMappableLocationOption()))); - await TestApp.AddTestData(context => - { - context.DataSetVersionMappings.Add(mappings); - }); + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); var client = BuildApp().CreateClient(); @@ -122,7 +95,7 @@ await TestApp.AddTestData(context => // Test that the mappings from the Controller are identical to the mappings saved in the database retrievedMappings.AssertDeepEqualTo( - mappings.LocationMappingPlan, + mapping.LocationMappingPlan, ignoreCollectionOrders: true); } @@ -163,40 +136,17 @@ private async Task GetLocationMappings( } public class ApplyBatchLocationMappingUpdatesTests( - TestApplicationFactory testApp) : DataSetVersionMappingControllerTests(testApp) + TestApplicationFactory testApp) + : DataSetVersionMappingControllerTests(testApp) { [Fact] public async Task Success() { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion currentDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 0) - .WithStatusPublished() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); - - DataSetVersion nextDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 1) - .WithStatusDraft() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); - - await TestApp.AddTestData(context => - { - context.DataSetVersions.AddRange(currentDataSetVersion, nextDataSetVersion); - context.DataSets.Update(dataSet); - }); + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() - .WithSourceDataSetVersionId(currentDataSetVersion.Id) + .WithSourceDataSetVersionId(initialDataSetVersion.Id) .WithTargetDataSetVersionId(nextDataSetVersion.Id) .WithLocationMappingPlan(DataFixture .DefaultLocationMappingPlan() @@ -240,19 +190,7 @@ await TestApp.AddTestData(context => targetKey: "target-location-1-key", candidate: DataFixture.DefaultMappableLocationOption()))); - await TestApp.AddTestData(context => - { - context.DataSetVersionMappings.Add(mappings); - }); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextDataSetVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile(FileType.Data)) - .WithPublicApiDataSetId(nextDataSetVersion.DataSetId) - .WithPublicApiDataSetVersion(nextDataSetVersion.SemVersion()); - - await TestApp.AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); List updates = [ @@ -277,16 +215,16 @@ await TestApp.AddTestData(context => var viewModel = response.AssertOk(); - var originalLocalAuthorityMappingToUpdate = mappings + var originalLocalAuthorityMappingToUpdate = mapping .GetLocationOptionMapping(GeographicLevel.LocalAuthority, "source-location-1-key"); - var originalLocalAuthorityMappingNotUpdated = mappings + var originalLocalAuthorityMappingNotUpdated = mapping .GetLocationOptionMapping(GeographicLevel.LocalAuthority, "source-location-2-key"); - var originalCountryMappingNotUpdated = mappings + var originalCountryMappingNotUpdated = mapping .GetLocationOptionMapping(GeographicLevel.Country, "source-location-1-key"); - var originalCountryMappingToUpdate = mappings + var originalCountryMappingToUpdate = mapping .GetLocationOptionMapping(GeographicLevel.Country, "source-location-3-key"); var expectedUpdateResponse = new BatchLocationMappingUpdatesResponseViewModel @@ -320,7 +258,7 @@ await TestApp.AddTestData(context => // that were updated. viewModel.AssertDeepEqualTo(expectedUpdateResponse, ignoreCollectionOrders: true); - var updatedMappings = await TestApp.GetDbContext() + var updatedMapping = await TestApp.GetDbContext() .DataSetVersionMappings .Include(m => m.TargetDataSetVersion) .SingleAsync(m => m.TargetDataSetVersionId == nextDataSetVersion.Id); @@ -344,7 +282,7 @@ originalLocalAuthorityMappingToUpdate with }, { "source-location-2-key", originalLocalAuthorityMappingNotUpdated } }, - Candidates = mappings + Candidates = mapping .LocationMappingPlan .Levels[GeographicLevel.LocalAuthority] .Candidates @@ -367,7 +305,7 @@ originalCountryMappingToUpdate with } } }, - Candidates = mappings + Candidates = mapping .LocationMappingPlan .Levels[GeographicLevel.Country] .Candidates @@ -377,18 +315,18 @@ originalCountryMappingToUpdate with // Test that the updated mappings retrieved from the database reflect the updates // that were requested. - updatedMappings.LocationMappingPlan.Levels.AssertDeepEqualTo( + updatedMapping.LocationMappingPlan.Levels.AssertDeepEqualTo( expectedFullMappings, ignoreCollectionOrders: true); // Assert that the batch saves still show the location mappings as incomplete, as there // are still mappings with type "AutoNone" in the plan. - Assert.False(updatedMappings.LocationMappingsComplete); + Assert.False(updatedMapping.LocationMappingsComplete); // Assert that this update constitutes a major version update, as some locations options // are 'ManualNone', indicating that some of the source location options may have been // removed thus creating a breaking change. - Assert.Equal("2.0.0", updatedMappings.TargetDataSetVersion.SemVersion()); + Assert.Equal("2.0.0", updatedMapping.TargetDataSetVersion.SemVersion().ToString()); } [Theory] @@ -406,35 +344,11 @@ public async Task Success_MappingsCompleteAndVersionUpdated( bool expectedMappingsComplete, string expectedVersion) { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion currentDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 0) - .WithStatusPublished() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); - - DataSetVersion nextDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 1) - .WithStatusDraft() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); - - await TestApp.AddTestData(context => - { - context.DataSetVersions.AddRange(currentDataSetVersion, nextDataSetVersion); - context.DataSets.Update(dataSet); - }); + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() - .WithSourceDataSetVersionId(currentDataSetVersion.Id) + .WithSourceDataSetVersionId(initialDataSetVersion.Id) .WithTargetDataSetVersionId(nextDataSetVersion.Id) .WithLocationMappingPlan(DataFixture .DefaultLocationMappingPlan() @@ -470,19 +384,7 @@ await TestApp.AddTestData(context => targetKey: "target-location-1-key", candidate: DataFixture.DefaultMappableLocationOption()))); - await TestApp.AddTestData(context => - { - context.DataSetVersionMappings.Add(mappings); - }); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextDataSetVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile()) - .WithPublicApiDataSetId(nextDataSetVersion.DataSetId) - .WithPublicApiDataSetVersion(nextDataSetVersion.SemVersion()); - - await TestApp.AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); List updates = [ @@ -503,54 +405,24 @@ await TestApp.AddTestData(context => response.AssertOk(); - var updatedMappings = await TestApp.GetDbContext() + var updatedMapping = await TestApp.GetDbContext() .DataSetVersionMappings .Include(m => m.TargetDataSetVersion) .SingleAsync(m => m.TargetDataSetVersionId == nextDataSetVersion.Id); - Assert.Equal(expectedMappingsComplete, updatedMappings.LocationMappingsComplete); - - Assert.Equal(expectedVersion, updatedMappings.TargetDataSetVersion.SemVersion()); + Assert.Equal(expectedMappingsComplete, updatedMapping.LocationMappingsComplete); - var updatedReleaseFile = await TestApp.GetDbContext() - .ReleaseFiles - .SingleAsync(rf => rf.PublicApiDataSetId == updatedMappings.TargetDataSetVersion.DataSetId); - - Assert.Equal(expectedVersion, updatedReleaseFile.PublicApiDataSetVersion); + await AssertCorrectDataSetVersionNumbers(updatedMapping, expectedVersion); } [Fact] public async Task Success_DeletedLevel_MajorUpdate() { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion currentDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 0) - .WithStatusPublished() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); - - DataSetVersion nextDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 1) - .WithStatusDraft() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); - - await TestApp.AddTestData(context => - { - context.DataSetVersions.AddRange(currentDataSetVersion, nextDataSetVersion); - context.DataSets.Update(dataSet); - }); + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() - .WithSourceDataSetVersionId(currentDataSetVersion.Id) + .WithSourceDataSetVersionId(initialDataSetVersion.Id) .WithTargetDataSetVersionId(nextDataSetVersion.Id) .WithLocationMappingPlan(DataFixture .DefaultLocationMappingPlan() @@ -567,7 +439,6 @@ await TestApp.AddTestData(context => .AddCandidate( targetKey: "target-location-1-key", candidate: DataFixture.DefaultMappableLocationOption())) - // Country level has been deleted and has no candidates. .AddLevel( level: GeographicLevel.Country, mappings: DataFixture @@ -579,19 +450,7 @@ await TestApp.AddTestData(context => .WithSource(DataFixture.DefaultMappableLocationOption()) .WithAutoNone()))); - await TestApp.AddTestData(context => - { - context.DataSetVersionMappings.Add(mappings); - }); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextDataSetVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile()) - .WithPublicApiDataSetId(nextDataSetVersion.DataSetId) - .WithPublicApiDataSetVersion(nextDataSetVersion.SemVersion()); - - await TestApp.AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); List updates = [ @@ -610,56 +469,26 @@ await TestApp.AddTestData(context => response.AssertOk(); - var updatedMappings = await TestApp.GetDbContext() + var updatedMapping = await TestApp.GetDbContext() .DataSetVersionMappings .Include(m => m.TargetDataSetVersion) .SingleAsync(m => m.TargetDataSetVersionId == nextDataSetVersion.Id); // This update completes the mapping but as there's a // location level deletion, it's a major version update. - Assert.True(updatedMappings.LocationMappingsComplete); - - Assert.Equal("2.0.0", updatedMappings.TargetDataSetVersion.SemVersion()); - - var updatedReleaseFile = await TestApp.GetDbContext() - .ReleaseFiles - .SingleAsync(rf => rf.PublicApiDataSetId == updatedMappings.TargetDataSetVersion.DataSetId); + Assert.True(updatedMapping.LocationMappingsComplete); - Assert.Equal("2.0.0", updatedReleaseFile.PublicApiDataSetVersion); + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); } [Fact] public async Task Success_AddedLevel_MinorUpdate() { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion currentDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 0) - .WithStatusPublished() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); - - DataSetVersion nextDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 1) - .WithStatusDraft() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); - await TestApp.AddTestData(context => - { - context.DataSetVersions.AddRange(currentDataSetVersion, nextDataSetVersion); - context.DataSets.Update(dataSet); - }); - - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() - .WithSourceDataSetVersionId(currentDataSetVersion.Id) + .WithSourceDataSetVersionId(initialDataSetVersion.Id) .WithTargetDataSetVersionId(nextDataSetVersion.Id) .WithLocationMappingPlan(DataFixture .DefaultLocationMappingPlan() @@ -686,19 +515,7 @@ await TestApp.AddTestData(context => candidate: DataFixture .DefaultMappableLocationOption()))); - await TestApp.AddTestData(context => - { - context.DataSetVersionMappings.Add(mappings); - }); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextDataSetVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile()) - .WithPublicApiDataSetId(nextDataSetVersion.DataSetId) - .WithPublicApiDataSetVersion(nextDataSetVersion.SemVersion()); - - await TestApp.AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); List updates = [ @@ -717,56 +534,26 @@ await TestApp.AddTestData(context => response.AssertOk(); - var updatedMappings = await TestApp.GetDbContext() + var updatedMapping = await TestApp.GetDbContext() .DataSetVersionMappings .Include(m => m.TargetDataSetVersion) .SingleAsync(m => m.TargetDataSetVersionId == nextDataSetVersion.Id); // This update completes the mapping as a location level was added // and isn't considered as needing to be mapped - minor version update. - Assert.True(updatedMappings.LocationMappingsComplete); - - Assert.Equal("1.1.0", updatedMappings.TargetDataSetVersion.SemVersion()); - - var updatedReleaseFile = await TestApp.GetDbContext() - .ReleaseFiles - .SingleAsync(rf => rf.PublicApiDataSetId == updatedMappings.TargetDataSetVersion.DataSetId); + Assert.True(updatedMapping.LocationMappingsComplete); - Assert.Equal("1.1.0", updatedReleaseFile.PublicApiDataSetVersion); + await AssertCorrectDataSetVersionNumbers(updatedMapping, "1.1.0"); } [Fact] public async Task SourceKeyDoesNotExist_Returns400_AndRollsBackTransaction() { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion currentDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 0) - .WithStatusPublished() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); - - DataSetVersion nextDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 1) - .WithStatusDraft() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); - - await TestApp.AddTestData(context => - { - context.DataSetVersions.AddRange(currentDataSetVersion, nextDataSetVersion); - context.DataSets.Update(dataSet); - }); + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() - .WithSourceDataSetVersionId(currentDataSetVersion.Id) + .WithSourceDataSetVersionId(initialDataSetVersion.Id) .WithTargetDataSetVersionId(nextDataSetVersion.Id) .WithLocationMappingPlan(DataFixture .DefaultLocationMappingPlan() @@ -801,19 +588,7 @@ await TestApp.AddTestData(context => targetKey: "target-country-location-1-key", candidate: DataFixture.DefaultMappableLocationOption()))); - await TestApp.AddTestData(context => - { - context.DataSetVersionMappings.Add(mappings); - }); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextDataSetVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile(FileType.Data)) - .WithPublicApiDataSetId(nextDataSetVersion.DataSetId) - .WithPublicApiDataSetVersion(nextDataSetVersion.SemVersion()); - - await TestApp.AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); List updates = [ @@ -867,42 +642,18 @@ await TestApp.AddTestData(context => // Test that the mappings are not updated due to the failures of some of the update requests. retrievedMappings.LocationMappingPlan.Levels.AssertDeepEqualTo( - mappings.LocationMappingPlan.Levels, + mapping.LocationMappingPlan.Levels, ignoreCollectionOrders: true); } [Fact] public async Task CandidateKeyDoesNotExist_Returns400() { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion currentDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 0) - .WithStatusPublished() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); - - DataSetVersion nextDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 1) - .WithStatusDraft() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); - - await TestApp.AddTestData(context => - { - context.DataSetVersions.AddRange(currentDataSetVersion, nextDataSetVersion); - context.DataSets.Update(dataSet); - }); + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() - .WithSourceDataSetVersionId(currentDataSetVersion.Id) + .WithSourceDataSetVersionId(initialDataSetVersion.Id) .WithTargetDataSetVersionId(nextDataSetVersion.Id) .WithLocationMappingPlan(DataFixture .DefaultLocationMappingPlan() @@ -941,10 +692,7 @@ await TestApp.AddTestData(context => targetKey: "target-country-location-1-key", candidate: DataFixture.DefaultMappableLocationOption()))); - await TestApp.AddTestData(context => - { - context.DataSetVersionMappings.Add(mappings); - }); + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); List updates = [ @@ -1006,7 +754,7 @@ await TestApp.AddTestData(context => // Test that the mappings are not updated due to the failures of some of the update requests. retrievedMappings.LocationMappingPlan.Levels.AssertDeepEqualTo( - mappings.LocationMappingPlan.Levels, + mapping.LocationMappingPlan.Levels, ignoreCollectionOrders: true); } @@ -1148,35 +896,11 @@ public class GetFilterMappingsTests( [Fact] public async Task Success() { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion currentDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 0) - .WithStatusPublished() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); - - DataSetVersion nextDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 1) - .WithStatusDraft() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); - - await TestApp.AddTestData(context => - { - context.DataSetVersions.AddRange(currentDataSetVersion, nextDataSetVersion); - context.DataSets.Update(dataSet); - }); + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() - .WithSourceDataSetVersionId(currentDataSetVersion.Id) + .WithSourceDataSetVersionId(initialDataSetVersion.Id) .WithTargetDataSetVersionId(nextDataSetVersion.Id) .WithFilterMappingPlan(DataFixture .DefaultFilterMappingPlan() @@ -1204,7 +928,7 @@ await TestApp.AddTestData(context => await TestApp.AddTestData(context => { - context.DataSetVersionMappings.Add(mappings); + context.DataSetVersionMappings.Add(mapping); }); var response = await GetFilterMappings( @@ -1214,7 +938,7 @@ await TestApp.AddTestData(context => // Test that the mappings from the Controller are identical to the mappings saved in the database retrievedMappings.AssertDeepEqualTo( - mappings.FilterMappingPlan, + mapping.FilterMappingPlan, ignoreCollectionOrders: true); } @@ -1256,35 +980,11 @@ public class ApplyBatchFilterOptionMappingUpdatesTests( [Fact] public async Task Success() { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion currentDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 0) - .WithStatusPublished() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); - - DataSetVersion nextDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 1) - .WithStatusDraft() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); - - await TestApp.AddTestData(context => - { - context.DataSetVersions.AddRange(currentDataSetVersion, nextDataSetVersion); - context.DataSets.Update(dataSet); - }); + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() - .WithSourceDataSetVersionId(currentDataSetVersion.Id) + .WithSourceDataSetVersionId(initialDataSetVersion.Id) .WithTargetDataSetVersionId(nextDataSetVersion.Id) .WithFilterMappingPlan(DataFixture .DefaultFilterMappingPlan() @@ -1312,19 +1012,7 @@ await TestApp.AddTestData(context => .AddOptionCandidate("filter-2-option-1-key", DataFixture .DefaultMappableFilterOption()))); - await TestApp.AddTestData(context => - { - context.DataSetVersionMappings.Add(mappings); - }); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextDataSetVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile(FileType.Data)) - .WithPublicApiDataSetId(nextDataSetVersion.DataSetId) - .WithPublicApiDataSetVersion(nextDataSetVersion.SemVersion()); - - await TestApp.AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); List updates = [ @@ -1357,7 +1045,7 @@ await TestApp.AddTestData(context => { FilterKey = "filter-1-key", SourceKey = "filter-1-option-1-key", - Mapping = mappings.GetFilterOptionMapping( + Mapping = mapping.GetFilterOptionMapping( filterKey: "filter-1-key", filterOptionKey: "filter-1-option-1-key") with { @@ -1369,7 +1057,7 @@ await TestApp.AddTestData(context => { FilterKey = "filter-2-key", SourceKey = "filter-2-option-1-key", - Mapping = mappings.GetFilterOptionMapping( + Mapping = mapping.GetFilterOptionMapping( filterKey: "filter-2-key", filterOptionKey: "filter-2-option-1-key") with { @@ -1384,22 +1072,22 @@ await TestApp.AddTestData(context => // that were updated. viewModel.AssertDeepEqualTo(expectedUpdateResponse, ignoreCollectionOrders: true); - var updatedMappings = await TestApp.GetDbContext() + var updatedMapping = await TestApp.GetDbContext() .DataSetVersionMappings - .Include(mapping => mapping.TargetDataSetVersion) + .Include(m => m.TargetDataSetVersion) .SingleAsync(m => m.TargetDataSetVersionId == nextDataSetVersion.Id); var expectedFullMappings = new Dictionary { { "filter-1-key", - mappings.GetFilterMapping("filter-1-key") with + mapping.GetFilterMapping("filter-1-key") with { OptionMappings = new Dictionary { { "filter-1-option-1-key", - mappings.GetFilterOptionMapping("filter-1-key", "filter-1-option-1-key") with + mapping.GetFilterOptionMapping("filter-1-key", "filter-1-option-1-key") with { Type = MappingType.ManualMapped, CandidateKey = "filter-1-option-1-key" @@ -1407,20 +1095,20 @@ await TestApp.AddTestData(context => }, { "filter-1-option-2-key", - mappings.GetFilterOptionMapping("filter-1-key", "filter-1-option-2-key") + mapping.GetFilterOptionMapping("filter-1-key", "filter-1-option-2-key") } } } }, { "filter-2-key", - mappings.GetFilterMapping("filter-2-key") with + mapping.GetFilterMapping("filter-2-key") with { OptionMappings = new Dictionary { { "filter-2-option-1-key", - mappings.GetFilterOptionMapping("filter-2-key", "filter-2-option-1-key") with + mapping.GetFilterOptionMapping("filter-2-key", "filter-2-option-1-key") with { Type = MappingType.ManualNone, CandidateKey = null @@ -1433,19 +1121,19 @@ await TestApp.AddTestData(context => // Test that the updated mappings retrieved from the database reflect the updates // that were requested. - updatedMappings.FilterMappingPlan.Mappings.AssertDeepEqualTo( + updatedMapping.FilterMappingPlan.Mappings.AssertDeepEqualTo( expectedFullMappings, ignoreCollectionOrders: true); // Assert that the batch saves show the filter mappings as complete, as there // are no remaining mappings with type "None" or "AutoNone" in the plan. - Assert.True(updatedMappings.FilterMappingsComplete); + Assert.True(updatedMapping.FilterMappingsComplete); // Assert that this update constitutes a major version update, as some filter options - // belonging to mapped filters have a mapping type of "ManualNone", indicating that + // belonging to mapped filters have a mapping type of "ManualNone", indicating that // some of the source filter options are no longer available in the target data set - // version, thus creating a breaking change. - Assert.Equal("2.0.0", updatedMappings.TargetDataSetVersion.SemVersion()); + // version, thus creating a breaking change. + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); } [Theory] @@ -1462,35 +1150,11 @@ public async Task Success_MappingsComplete( MappingType unchangedMappingType, bool expectedMappingsComplete) { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion currentDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 0) - .WithStatusPublished() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); - - DataSetVersion nextDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 1) - .WithStatusDraft() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); - await TestApp.AddTestData(context => - { - context.DataSetVersions.AddRange(currentDataSetVersion, nextDataSetVersion); - context.DataSets.Update(dataSet); - }); - - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() - .WithSourceDataSetVersionId(currentDataSetVersion.Id) + .WithSourceDataSetVersionId(initialDataSetVersion.Id) .WithTargetDataSetVersionId(nextDataSetVersion.Id) .WithFilterMappingPlan(DataFixture .DefaultFilterMappingPlan() @@ -1511,7 +1175,7 @@ await TestApp.AddTestData(context => MappingType.ManualMapped or MappingType.AutoMapped => "filter-2-option-1-key", _ => null }))) - // Add an unmappable filter and filter options. Because we don't currently allow the + // Add an unmappable filter and filter options. Because we don't currently allow the // users to update mappings for filters, this should not count against the calculation // of the FilterMappingsComplete flag. .AddFilterMapping("filter-3-key", DataFixture @@ -1529,19 +1193,7 @@ await TestApp.AddTestData(context => .AddOptionCandidate("filter-2-option-1-key", DataFixture .DefaultMappableFilterOption()))); - await TestApp.AddTestData(context => - { - context.DataSetVersionMappings.Add(mappings); - }); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextDataSetVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile(FileType.Data)) - .WithPublicApiDataSetId(nextDataSetVersion.DataSetId) - .WithPublicApiDataSetVersion(nextDataSetVersion.SemVersion()); - - await TestApp.AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); var mappingCandidateKey = updatedMappingType == MappingType.ManualMapped ? "filter-1-option-1-key" @@ -1564,13 +1216,13 @@ await TestApp.AddTestData(context => response.AssertOk(); - var updatedMappings = await TestApp.GetDbContext() + var updatedMapping = await TestApp.GetDbContext() .DataSetVersionMappings .SingleAsync(m => m.TargetDataSetVersionId == nextDataSetVersion.Id); // Assert that the batch save calculates the LocationMappingsComplete flag as expected given the // combination of the requested mapping update and the existing mapping that is untouched. - Assert.Equal(expectedMappingsComplete, updatedMappings.FilterMappingsComplete); + Assert.Equal(expectedMappingsComplete, updatedMapping.FilterMappingsComplete); } [Theory] @@ -1587,35 +1239,11 @@ public async Task Success_VersionUpdate( MappingType unchangedMappingType, string expectedVersion) { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion currentDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 0) - .WithStatusPublished() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); - - DataSetVersion nextDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 1) - .WithStatusDraft() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); - await TestApp.AddTestData(context => - { - context.DataSetVersions.AddRange(currentDataSetVersion, nextDataSetVersion); - context.DataSets.Update(dataSet); - }); - - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() - .WithSourceDataSetVersionId(currentDataSetVersion.Id) + .WithSourceDataSetVersionId(initialDataSetVersion.Id) .WithTargetDataSetVersionId(nextDataSetVersion.Id) .WithFilterMappingPlan(DataFixture .DefaultFilterMappingPlan() @@ -1645,19 +1273,7 @@ await TestApp.AddTestData(context => .AddOptionCandidate("filter-2-option-1-key", DataFixture .DefaultMappableFilterOption()))); - await TestApp.AddTestData(context => - { - context.DataSetVersionMappings.Add(mappings); - }); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextDataSetVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile(FileType.Data)) - .WithPublicApiDataSetId(nextDataSetVersion.DataSetId) - .WithPublicApiDataSetVersion(nextDataSetVersion.SemVersion()); - - await TestApp.AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); var mappingCandidateKey = updatedMappingType == MappingType.ManualMapped ? "filter-1-option-1-key" @@ -1680,19 +1296,12 @@ await TestApp.AddTestData(context => response.AssertOk(); - var updatedMappings = await TestApp.GetDbContext() + var updatedMapping = await TestApp.GetDbContext() .DataSetVersionMappings .Include(m => m.TargetDataSetVersion) .SingleAsync(m => m.TargetDataSetVersionId == nextDataSetVersion.Id); - // Assert that the batch save calculates the next version number as expected. - Assert.Equal(expectedVersion, updatedMappings.TargetDataSetVersion.SemVersion()); - - var updatedReleaseFile = await TestApp.GetDbContext() - .ReleaseFiles - .SingleAsync(rf => rf.PublicApiDataSetId == updatedMappings.TargetDataSetVersion.DataSetId); - - Assert.Equal(expectedVersion, updatedReleaseFile.PublicApiDataSetVersion); + await AssertCorrectDataSetVersionNumbers(updatedMapping, expectedVersion); } [Theory] @@ -1700,35 +1309,11 @@ await TestApp.AddTestData(context => [InlineData(MappingType.ManualNone)] public async Task Success_VersionUpdates_UnmappableFilter(MappingType updatedMappingType) { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion currentDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 0) - .WithStatusPublished() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); - - DataSetVersion nextDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 1) - .WithStatusDraft() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); - await TestApp.AddTestData(context => - { - context.DataSetVersions.AddRange(currentDataSetVersion, nextDataSetVersion); - context.DataSets.Update(dataSet); - }); - - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() - .WithSourceDataSetVersionId(currentDataSetVersion.Id) + .WithSourceDataSetVersionId(initialDataSetVersion.Id) .WithTargetDataSetVersionId(nextDataSetVersion.Id) .WithFilterMappingPlan(DataFixture .DefaultFilterMappingPlan() @@ -1757,19 +1342,7 @@ await TestApp.AddTestData(context => .AddOptionCandidate("filter-2-option-1-key", DataFixture .DefaultMappableFilterOption()))); - await TestApp.AddTestData(context => - { - context.DataSetVersionMappings.Add(mappings); - }); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextDataSetVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile(FileType.Data)) - .WithPublicApiDataSetId(nextDataSetVersion.DataSetId) - .WithPublicApiDataSetVersion(nextDataSetVersion.SemVersion()); - - await TestApp.AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); var mappingCandidateKey = updatedMappingType == MappingType.ManualMapped ? "filter-1-option-1-key" @@ -1792,49 +1365,25 @@ await TestApp.AddTestData(context => response.AssertOk(); - var updatedMappings = await TestApp.GetDbContext() + var updatedMapping = await TestApp.GetDbContext() .DataSetVersionMappings .Include(m => m.TargetDataSetVersion) .SingleAsync(m => m.TargetDataSetVersionId == nextDataSetVersion.Id); // Assert that the batch save calculates the next version number as a major change, // as filter options that were in the source data set version no longer appear in the - // next version. - Assert.Equal("2.0.0", updatedMappings.TargetDataSetVersion.SemVersion()); + // next version. + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); } [Fact] public async Task SourceKeyDoesNotExist_Returns400_AndRollsBackTransaction() { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion currentDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 0) - .WithStatusPublished() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); - - DataSetVersion nextDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 1) - .WithStatusDraft() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); - await TestApp.AddTestData(context => - { - context.DataSetVersions.AddRange(currentDataSetVersion, nextDataSetVersion); - context.DataSets.Update(dataSet); - }); - - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() - .WithSourceDataSetVersionId(currentDataSetVersion.Id) + .WithSourceDataSetVersionId(initialDataSetVersion.Id) .WithTargetDataSetVersionId(nextDataSetVersion.Id) .WithFilterMappingPlan(DataFixture .DefaultFilterMappingPlan() @@ -1856,10 +1405,7 @@ await TestApp.AddTestData(context => .AddOptionCandidate("filter-1-option-3-key", DataFixture .DefaultMappableFilterOption()))); - await TestApp.AddTestData(context => - { - context.DataSetVersionMappings.Add(mappings); - }); + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); List updates = [ @@ -1901,42 +1447,18 @@ await TestApp.AddTestData(context => // Test that the mappings are not updated due to the failures of some of the update requests. retrievedMappings.FilterMappingPlan.Mappings.AssertDeepEqualTo( - mappings.FilterMappingPlan.Mappings, + mapping.FilterMappingPlan.Mappings, ignoreCollectionOrders: true); } [Fact] public async Task CandidateKeyDoesNotExist_Returns400() { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion currentDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 0) - .WithStatusPublished() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); - - DataSetVersion nextDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 1) - .WithStatusDraft() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); - await TestApp.AddTestData(context => - { - context.DataSetVersions.AddRange(currentDataSetVersion, nextDataSetVersion); - context.DataSets.Update(dataSet); - }); - - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() - .WithSourceDataSetVersionId(currentDataSetVersion.Id) + .WithSourceDataSetVersionId(initialDataSetVersion.Id) .WithTargetDataSetVersionId(nextDataSetVersion.Id) .WithFilterMappingPlan(DataFixture .DefaultFilterMappingPlan() @@ -1966,10 +1488,7 @@ await TestApp.AddTestData(context => .AddOptionCandidate("filter-2-option-1-key", DataFixture .DefaultMappableFilterOption()))); - await TestApp.AddTestData(context => - { - context.DataSetVersionMappings.Add(mappings); - }); + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); List updates = [ @@ -2031,42 +1550,18 @@ await TestApp.AddTestData(context => // Test that the mappings are not updated due to the failures of some of the update requests. retrievedMappings.FilterMappingPlan.Mappings.AssertDeepEqualTo( - mappings.FilterMappingPlan.Mappings, + mapping.FilterMappingPlan.Mappings, ignoreCollectionOrders: true); } [Fact] public async Task OwningFilterNotMapped_Returns400() { - DataSet dataSet = DataFixture - .DefaultDataSet() - .WithStatusPublished(); - - await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); - - DataSetVersion currentDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 0) - .WithStatusPublished() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); - - DataSetVersion nextDataSetVersion = DataFixture - .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) - .WithVersionNumber(major: 1, minor: 1) - .WithStatusDraft() - .WithDataSet(dataSet) - .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); - await TestApp.AddTestData(context => - { - context.DataSetVersions.AddRange(currentDataSetVersion, nextDataSetVersion); - context.DataSets.Update(dataSet); - }); - - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() - .WithSourceDataSetVersionId(currentDataSetVersion.Id) + .WithSourceDataSetVersionId(initialDataSetVersion.Id) .WithTargetDataSetVersionId(nextDataSetVersion.Id) .WithFilterMappingPlan(DataFixture .DefaultFilterMappingPlan() @@ -2084,10 +1579,7 @@ await TestApp.AddTestData(context => .AddOptionCandidate("filter-1-option-1-key", DataFixture .DefaultMappableFilterOption()))); - await TestApp.AddTestData(context => - { - context.DataSetVersionMappings.Add(mappings); - }); + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); List updates = [ @@ -2122,7 +1614,7 @@ await TestApp.AddTestData(context => // Test that the mappings are not updated due to the failures of some of the update requests. retrievedMappings.FilterMappingPlan.Mappings.AssertDeepEqualTo( - mappings.FilterMappingPlan.Mappings, + mapping.FilterMappingPlan.Mappings, ignoreCollectionOrders: true); } @@ -2264,6 +1756,57 @@ private async Task ApplyBatchFilterOptionMappingUpdates( } } + private async Task AssertCorrectDataSetVersionNumbers(DataSetVersionMapping mapping, string expectedVersion) + { + Assert.Equal(expectedVersion, mapping.TargetDataSetVersion.SemVersion().ToString()); + + var updatedReleaseFile = await TestApp.GetDbContext() + .ReleaseFiles + .SingleAsync(rf => rf.PublicApiDataSetId == mapping.TargetDataSetVersion.DataSetId); + + Assert.Equal(expectedVersion, updatedReleaseFile.PublicApiDataSetVersion?.ToString()); + } + + private async Task<(DataSetVersion initialVersion, DataSetVersion nextVersion)> CreateInitialAndNextDataSetVersion() + { + DataSet dataSet = DataFixture + .DefaultDataSet() + .WithStatusPublished(); + + await TestApp.AddTestData(context => context.DataSets.Add(dataSet)); + + DataSetVersion initialDataSetVersion = DataFixture + .DefaultDataSetVersion(filters: 1, indicators: 1, locations: 1, timePeriods: 2) + .WithVersionNumber(major: 1, minor: 0) + .WithStatusPublished() + .WithDataSet(dataSet) + .FinishWith(dsv => dsv.DataSet.LatestLiveVersion = dsv); + + DataSetVersion nextDataSetVersion = DataFixture + .DefaultDataSetVersion() + .WithVersionNumber(major: 1, minor: 1) + .WithStatusDraft() + .WithDataSet(dataSet) + .FinishWith(dsv => dsv.DataSet.LatestDraftVersion = dsv); + + await TestApp.AddTestData(context => + { + context.DataSetVersions.AddRange(initialDataSetVersion, nextDataSetVersion); + context.DataSets.Update(dataSet); + }); + + ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() + .WithId(nextDataSetVersion.Release.ReleaseFileId) + .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) + .WithFile(DataFixture.DefaultFile(FileType.Data)) + .WithPublicApiDataSetId(nextDataSetVersion.DataSetId) + .WithPublicApiDataSetVersion(nextDataSetVersion.SemVersion()); + + await TestApp.AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + + return (initialDataSetVersion, nextDataSetVersion); + } + private WebApplicationFactory BuildApp(ClaimsPrincipal? user = null) { return TestApp.SetUser(user ?? DataFixture.BauUser()); From 1d068ce45a009ba9efd9095a622506cf4f4de04e Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Sat, 12 Oct 2024 02:01:41 +0100 Subject: [PATCH 60/80] EES-5570 Cleanup `ProcessNextDataSetVersionMappingsFunctionTests` - Renames `mappings` variables to singular `mapping` - Refactors creation of release files into the common `CreateNextDataSetVersionAndDataFiles` method. --- ...NextDataSetVersionMappingsFunctionTests.cs | 414 ++++++------------ 1 file changed, 138 insertions(+), 276 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs index fba8459ea1a..e41e3027bc1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs @@ -148,7 +148,7 @@ public async Task Success_MetaSummary() await CreateMappings(instanceId); - var updatedDataSetVersion = GetDataSetVersion(nextVersion); + var updatedDataSetVersion = await GetDataSetVersion(nextVersion); // Assert that the MetaSummary has been generated correctly from the CSV. var metaSummary = updatedDataSetVersion.MetaSummary; @@ -242,11 +242,11 @@ await AddTestData(context => await CreateMappings(instanceId); - var mappings = GetDataSetVersionMapping(nextVersion); + var mapping = await GetDataSetVersionMapping(nextVersion); - Assert.Equal(initialVersion.Id, mappings.SourceDataSetVersionId); - Assert.Equal(nextVersion.Id, mappings.TargetDataSetVersionId); - Assert.False(mappings.LocationMappingsComplete); + Assert.Equal(initialVersion.Id, mapping.SourceDataSetVersionId); + Assert.Equal(nextVersion.Id, mapping.TargetDataSetVersionId); + Assert.False(mapping.LocationMappingsComplete); var expectedLocationMappingsFromSource = initialLocationMeta .ToDictionary( @@ -280,12 +280,12 @@ await AddTestData(context => .ExpectedGeographicLevels .Concat([GeographicLevel.Provider]) .Order(), - mappings.LocationMappingPlan + mapping.LocationMappingPlan .Levels .Select(level => level.Key) .Order()); - mappings.LocationMappingPlan.Levels.ForEach(level => + mapping.LocationMappingPlan.Levels.ForEach(level => { var matchingLevelFromSource = expectedLocationMappingsFromSource.GetValueOrDefault(level.Key); @@ -310,10 +310,10 @@ public async Task Success_Candidates() await CreateMappings(instanceId); - var mappings = GetDataSetVersionMapping(nextVersion); + var mapping = await GetDataSetVersionMapping(nextVersion); - Assert.Equal(initialVersion.Id, mappings.SourceDataSetVersionId); - Assert.Equal(nextVersion.Id, mappings.TargetDataSetVersionId); + Assert.Equal(initialVersion.Id, mapping.SourceDataSetVersionId); + Assert.Equal(nextVersion.Id, mapping.TargetDataSetVersionId); var expectedLocationLevels = ProcessorTestData .AbsenceSchool @@ -338,7 +338,7 @@ public async Task Success_Candidates() }) }); - mappings.LocationMappingPlan.Levels.AssertDeepEqualTo( + mapping.LocationMappingPlan.Levels.AssertDeepEqualTo( expectedLocationLevels, ignoreCollectionOrders: true); } @@ -367,11 +367,11 @@ await AddTestData(context => await CreateMappings(instanceId); - var mappings = GetDataSetVersionMapping(nextVersion); + var mapping = await GetDataSetVersionMapping(nextVersion); - Assert.Equal(initialVersion.Id, mappings.SourceDataSetVersionId); - Assert.Equal(nextVersion.Id, mappings.TargetDataSetVersionId); - Assert.False(mappings.FilterMappingsComplete); + Assert.Equal(initialVersion.Id, mapping.SourceDataSetVersionId); + Assert.Equal(nextVersion.Id, mapping.TargetDataSetVersionId); + Assert.False(mapping.FilterMappingsComplete); var expectedFilterMappings = initialFilterMeta .ToDictionary( @@ -395,7 +395,7 @@ await AddTestData(context => }) }); - mappings.FilterMappingPlan.Mappings.AssertDeepEqualTo( + mapping.FilterMappingPlan.Mappings.AssertDeepEqualTo( expectedFilterMappings, ignoreCollectionOrders: true); } @@ -408,10 +408,10 @@ public async Task Success_Candidates() await CreateMappings(instanceId); - var mappings = GetDataSetVersionMapping(nextVersion); + var mapping = await GetDataSetVersionMapping(nextVersion); - Assert.Equal(initialVersion.Id, mappings.SourceDataSetVersionId); - Assert.Equal(nextVersion.Id, mappings.TargetDataSetVersionId); + Assert.Equal(initialVersion.Id, mapping.SourceDataSetVersionId); + Assert.Equal(nextVersion.Id, mapping.TargetDataSetVersionId); var expectedFilterTargets = ProcessorTestData .AbsenceSchool @@ -430,7 +430,7 @@ public async Task Success_Candidates() new MappableFilterOption { Label = optionMeta.Label }) }); - mappings.FilterMappingPlan.Candidates.AssertDeepEqualTo( + mapping.FilterMappingPlan.Candidates.AssertDeepEqualTo( expectedFilterTargets, ignoreCollectionOrders: true); } @@ -468,15 +468,6 @@ await AddTestData(context => FilterMappingPlan = new FilterMappingPlan() })); - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile()) - .WithPublicApiDataSetId(nextVersion.DataSetId) - .WithPublicApiDataSetVersion(nextVersion.SemVersion()); - - await AddTestData(context => context.ReleaseFiles.Add(releaseFile)); - await ApplyAutoMappings(instanceId); var savedImport = await GetDbContext() @@ -499,7 +490,7 @@ public async Task Incomplete_UnmappedOptionWithNoCandidate_MajorUpdate() var (instanceId, originalVersion, nextVersion) = await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() .WithSourceDataSetVersionId(originalVersion.Id) .WithTargetDataSetVersionId(nextVersion.Id) @@ -528,32 +519,23 @@ public async Task Incomplete_UnmappedOptionWithNoCandidate_MajorUpdate() targetKey: "la-location-1-key", candidate: DataFixture.DefaultMappableLocationOption()))); - await AddTestData(context => context.DataSetVersionMappings.Add(mappings)); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile()) - .WithPublicApiDataSetId(nextVersion.DataSetId) - .WithPublicApiDataSetVersion(nextVersion.SemVersion()); - - await AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); await ApplyAutoMappings(instanceId); - var updatedMappings = GetDataSetVersionMapping(nextVersion); + var updatedMapping = await GetDataSetVersionMapping(nextVersion); - var laMapping1 = mappings + var laMapping1 = mapping .GetLocationOptionMapping(GeographicLevel.LocalAuthority, "la-location-1-key"); - var laMapping2 = mappings + var laMapping2 = mapping .GetLocationOptionMapping(GeographicLevel.LocalAuthority, "la-location-2-key"); Dictionary expectedLevelMappings = new() { { GeographicLevel.LocalAuthority, - mappings.GetLocationLevelMappings(GeographicLevel.LocalAuthority) with + mapping.GetLocationLevelMappings(GeographicLevel.LocalAuthority) with { Mappings = new Dictionary { @@ -576,20 +558,14 @@ public async Task Incomplete_UnmappedOptionWithNoCandidate_MajorUpdate() }, }; - updatedMappings.LocationMappingPlan.Levels.AssertDeepEqualTo( + updatedMapping.LocationMappingPlan.Levels.AssertDeepEqualTo( expectedLevelMappings, ignoreCollectionOrders: true); - Assert.False(updatedMappings.LocationMappingsComplete); + Assert.False(updatedMapping.LocationMappingsComplete); // Some options have no auto-mapped candidate - major version update. - Assert.Equal("2.0.0", updatedMappings.TargetDataSetVersion.SemVersion()); - - var updatedReleaseFile = await GetDbContext() - .ReleaseFiles - .SingleAsync(rf => rf.PublicApiDataSetId == updatedMappings.TargetDataSetVersion.DataSetId); - - Assert.Equal(updatedMappings.TargetDataSetVersion.SemVersion(), updatedReleaseFile.PublicApiDataSetVersion); + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); } [Fact] @@ -598,7 +574,7 @@ public async Task Incomplete_UnmappedOptionWithNewCandidate_MajorUpdate() var (instanceId, originalVersion, nextVersion) = await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() .WithSourceDataSetVersionId(originalVersion.Id) .WithTargetDataSetVersionId(nextVersion.Id) @@ -631,32 +607,23 @@ public async Task Incomplete_UnmappedOptionWithNewCandidate_MajorUpdate() targetKey: "la-location-3-key", candidate: DataFixture.DefaultMappableLocationOption()))); - await AddTestData(context => context.DataSetVersionMappings.Add(mappings)); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile()) - .WithPublicApiDataSetId(nextVersion.DataSetId) - .WithPublicApiDataSetVersion(nextVersion.SemVersion()); - - await AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); await ApplyAutoMappings(instanceId); - var updatedMappings = GetDataSetVersionMapping(nextVersion); + var updatedMapping = await GetDataSetVersionMapping(nextVersion); - var laMapping1 = mappings + var laMapping1 = mapping .GetLocationOptionMapping(GeographicLevel.LocalAuthority, "la-location-1-key"); - var laMapping2 = mappings + var laMapping2 = mapping .GetLocationOptionMapping(GeographicLevel.LocalAuthority, "la-location-2-key"); Dictionary expectedLevelMappings = new() { { GeographicLevel.LocalAuthority, - mappings.GetLocationLevelMappings(GeographicLevel.LocalAuthority) with + mapping.GetLocationLevelMappings(GeographicLevel.LocalAuthority) with { Mappings = new Dictionary { @@ -679,20 +646,14 @@ public async Task Incomplete_UnmappedOptionWithNewCandidate_MajorUpdate() }, }; - updatedMappings.LocationMappingPlan.Levels.AssertDeepEqualTo( + updatedMapping.LocationMappingPlan.Levels.AssertDeepEqualTo( expectedLevelMappings, ignoreCollectionOrders: true); - Assert.False(updatedMappings.LocationMappingsComplete); + Assert.False(updatedMapping.LocationMappingsComplete); // Some options have no auto-mapped candidate - major version update. - Assert.Equal("2.0.0", updatedMappings.TargetDataSetVersion.SemVersion()); - - var updatedReleaseFile = await GetDbContext() - .ReleaseFiles - .SingleAsync(rf => rf.PublicApiDataSetId == updatedMappings.TargetDataSetVersion.DataSetId); - - Assert.Equal(updatedMappings.TargetDataSetVersion.SemVersion(), updatedReleaseFile.PublicApiDataSetVersion); + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); } [Fact] @@ -701,7 +662,7 @@ public async Task Complete_DeletedLevel_MajorUpdate() var (instanceId, originalVersion, nextVersion) = await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() .WithSourceDataSetVersionId(originalVersion.Id) .WithTargetDataSetVersionId(nextVersion.Id) @@ -718,34 +679,25 @@ public async Task Complete_DeletedLevel_MajorUpdate() .WithSource(DataFixture.DefaultMappableLocationOption()) .WithManualNone()))); - await AddTestData(context => context.DataSetVersionMappings.Add(mappings)); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile()) - .WithPublicApiDataSetId(nextVersion.DataSetId) - .WithPublicApiDataSetVersion(nextVersion.SemVersion()); - - await AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); await ApplyAutoMappings(instanceId); - var updatedMappings = GetDataSetVersionMapping(nextVersion); + var updatedMapping = await GetDataSetVersionMapping(nextVersion); - var rscMapping1 = mappings + var rscMapping = mapping .GetLocationOptionMapping(GeographicLevel.RscRegion, "rsc-location-1-key"); Dictionary expectedLevelMappings = new() { { GeographicLevel.RscRegion, - mappings.GetLocationLevelMappings(GeographicLevel.RscRegion) with + mapping.GetLocationLevelMappings(GeographicLevel.RscRegion) with { Mappings = new Dictionary { { - "rsc-location-1-key", rscMapping1 with + "rsc-location-1-key", rscMapping with { Type = MappingType.AutoNone, CandidateKey = null @@ -756,20 +708,14 @@ public async Task Complete_DeletedLevel_MajorUpdate() }, }; - updatedMappings.LocationMappingPlan.Levels.AssertDeepEqualTo( + updatedMapping.LocationMappingPlan.Levels.AssertDeepEqualTo( expectedLevelMappings, ignoreCollectionOrders: true); - Assert.True(updatedMappings.LocationMappingsComplete); + Assert.True(updatedMapping.LocationMappingsComplete); // Level has been deleted (cannot be mapped) - major version update. - Assert.Equal("2.0.0", updatedMappings.TargetDataSetVersion.SemVersion()); - - var updatedReleaseFile = await GetDbContext() - .ReleaseFiles - .SingleAsync(rf => rf.PublicApiDataSetId == updatedMappings.TargetDataSetVersion.DataSetId); - - Assert.Equal(updatedMappings.TargetDataSetVersion.SemVersion(), updatedReleaseFile.PublicApiDataSetVersion); + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); } [Fact] @@ -778,7 +724,7 @@ public async Task Complete_NewLevel_MinorUpdate() var (instanceId, originalVersion, nextVersion) = await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() .WithSourceDataSetVersionId(originalVersion.Id) .WithTargetDataSetVersionId(nextVersion.Id) @@ -793,40 +739,25 @@ public async Task Complete_NewLevel_MinorUpdate() candidate: DataFixture .DefaultMappableLocationOption()))); - await AddTestData(context => context.DataSetVersionMappings.Add(mappings)); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile()) - .WithPublicApiDataSetId(nextVersion.DataSetId) - .WithPublicApiDataSetVersion(nextVersion.SemVersion()); - - await AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); await ApplyAutoMappings(instanceId); - var updatedMappings = GetDataSetVersionMapping(nextVersion); + var updatedMapping = await GetDataSetVersionMapping(nextVersion); Dictionary expectedLevelMappings = new() { - { GeographicLevel.Country, mappings.GetLocationLevelMappings(GeographicLevel.Country) } + { GeographicLevel.Country, mapping.GetLocationLevelMappings(GeographicLevel.Country) } }; - updatedMappings.LocationMappingPlan.Levels.AssertDeepEqualTo( + updatedMapping.LocationMappingPlan.Levels.AssertDeepEqualTo( expectedLevelMappings, ignoreCollectionOrders: true); - Assert.True(updatedMappings.LocationMappingsComplete); + Assert.True(updatedMapping.LocationMappingsComplete); // Level has been added - minor version update. - Assert.Equal("1.1.0", updatedMappings.TargetDataSetVersion.SemVersion()); - - var updatedReleaseFile = await GetDbContext() - .ReleaseFiles - .SingleAsync(rf => rf.PublicApiDataSetId == updatedMappings.TargetDataSetVersion.DataSetId); - - Assert.Equal(updatedMappings.TargetDataSetVersion.SemVersion(), updatedReleaseFile.PublicApiDataSetVersion); + await AssertCorrectDataSetVersionNumbers(updatedMapping, "1.1.0"); } [Fact] @@ -835,7 +766,7 @@ public async Task Complete_ExactMatch_MinorUpdate() var (instanceId, originalVersion, nextVersion) = await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() .WithSourceDataSetVersionId(originalVersion.Id) .WithTargetDataSetVersionId(nextVersion.Id) @@ -855,29 +786,20 @@ public async Task Complete_ExactMatch_MinorUpdate() targetKey: "location-1-key", candidate: DataFixture.DefaultMappableLocationOption()))); - await AddTestData(context => context.DataSetVersionMappings.Add(mappings)); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile()) - .WithPublicApiDataSetId(nextVersion.DataSetId) - .WithPublicApiDataSetVersion(nextVersion.SemVersion()); - - await AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); await ApplyAutoMappings(instanceId); - var updatedMappings = GetDataSetVersionMapping(nextVersion); + var updatedMapping = await GetDataSetVersionMapping(nextVersion); - var originalLocationMapping = mappings + var originalLocationMapping = mapping .GetLocationOptionMapping(GeographicLevel.LocalAuthority, "location-1-key"); Dictionary expectedLevelMappings = new() { { GeographicLevel.LocalAuthority, - mappings.GetLocationLevelMappings(GeographicLevel.LocalAuthority) with + mapping.GetLocationLevelMappings(GeographicLevel.LocalAuthority) with { Mappings = new Dictionary { @@ -893,18 +815,12 @@ public async Task Complete_ExactMatch_MinorUpdate() } }; - updatedMappings.LocationMappingPlan.Levels.AssertDeepEqualTo(expectedLevelMappings); + updatedMapping.LocationMappingPlan.Levels.AssertDeepEqualTo(expectedLevelMappings); - Assert.True(updatedMappings.LocationMappingsComplete); + Assert.True(updatedMapping.LocationMappingsComplete); // All source options have auto-mapped candidates - minor version update. - Assert.Equal("1.1.0", updatedMappings.TargetDataSetVersion.SemVersion()); - - var updatedReleaseFile = await GetDbContext() - .ReleaseFiles - .SingleAsync(rf => rf.PublicApiDataSetId == updatedMappings.TargetDataSetVersion.DataSetId); - - Assert.Equal(updatedMappings.TargetDataSetVersion.SemVersion(), updatedReleaseFile.PublicApiDataSetVersion); + await AssertCorrectDataSetVersionNumbers(updatedMapping, "1.1.0"); } [Fact] @@ -913,7 +829,7 @@ public async Task Complete_AutoMappedAndNewOptions_MinorUpdate() var (instanceId, originalVersion, nextVersion) = await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() .WithSourceDataSetVersionId(originalVersion.Id) .WithTargetDataSetVersionId(nextVersion.Id) @@ -947,29 +863,20 @@ public async Task Complete_AutoMappedAndNewOptions_MinorUpdate() candidate: DataFixture .DefaultMappableLocationOption()))); - await AddTestData(context => context.DataSetVersionMappings.Add(mappings)); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile()) - .WithPublicApiDataSetId(nextVersion.DataSetId) - .WithPublicApiDataSetVersion(nextVersion.SemVersion()); - - await AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); await ApplyAutoMappings(instanceId); - var updatedMappings = GetDataSetVersionMapping(nextVersion); + var updatedMapping = await GetDataSetVersionMapping(nextVersion); - var originalLaMapping = mappings + var originalLaMapping = mapping .GetLocationOptionMapping(GeographicLevel.LocalAuthority, "la-location-1-key"); Dictionary expectedLevelMappings = new() { { GeographicLevel.LocalAuthority, - mappings.GetLocationLevelMappings(GeographicLevel.LocalAuthority) with + mapping.GetLocationLevelMappings(GeographicLevel.LocalAuthority) with { Mappings = new Dictionary { @@ -983,23 +890,17 @@ public async Task Complete_AutoMappedAndNewOptions_MinorUpdate() } } }, - { GeographicLevel.RscRegion, mappings.GetLocationLevelMappings(GeographicLevel.RscRegion) } + { GeographicLevel.RscRegion, mapping.GetLocationLevelMappings(GeographicLevel.RscRegion) } }; - updatedMappings.LocationMappingPlan.Levels.AssertDeepEqualTo( + updatedMapping.LocationMappingPlan.Levels.AssertDeepEqualTo( expectedLevelMappings, ignoreCollectionOrders: true); - Assert.True(updatedMappings.LocationMappingsComplete); + Assert.True(updatedMapping.LocationMappingsComplete); // All source options have auto-mapped candidates - minor version update. - Assert.Equal("1.1.0", updatedMappings.TargetDataSetVersion.SemVersion()); - - var updatedReleaseFile = await GetDbContext() - .ReleaseFiles - .SingleAsync(rf => rf.PublicApiDataSetId == updatedMappings.TargetDataSetVersion.DataSetId); - - Assert.Equal(updatedMappings.TargetDataSetVersion.SemVersion(), updatedReleaseFile.PublicApiDataSetVersion); + await AssertCorrectDataSetVersionNumbers(updatedMapping, "1.1.0"); } } @@ -1016,7 +917,7 @@ public async Task PartiallyComplete() // Create a mapping plan based on 2 data set versions with partially overlapping filters. // Both have "Filter 1" and both have "Filter 1 option 1", but then each also contains Filter 1 // options that the other do not, and each also contains filters that the other does not. - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() .WithSourceDataSetVersionId(originalVersion.Id) .WithTargetDataSetVersionId(nextVersion.Id) @@ -1044,25 +945,16 @@ public async Task PartiallyComplete() .AddOptionCandidate("filter-1-option-3-key", DataFixture .DefaultMappableFilterOption()))); - await AddTestData(context => context.DataSetVersionMappings.Add(mappings)); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile()) - .WithPublicApiDataSetId(nextVersion.DataSetId) - .WithPublicApiDataSetVersion(nextVersion.SemVersion()); - - await AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); await ApplyAutoMappings(instanceId); - var updatedMappings = GetDataSetVersionMapping(nextVersion); + var updatedMapping = await GetDataSetVersionMapping(nextVersion); Dictionary expectedFilterMappings = new() { { - "filter-1-key", mappings.GetFilterMapping("filter-1-key") with + "filter-1-key", mapping.GetFilterMapping("filter-1-key") with { // The code managed to establish an automapping for this filter. Type = MappingType.AutoMapped, @@ -1071,7 +963,7 @@ public async Task PartiallyComplete() { { "filter-1-option-1-key", - mappings.GetFilterOptionMapping("filter-1-key", "filter-1-option-1-key") with + mapping.GetFilterOptionMapping("filter-1-key", "filter-1-option-1-key") with { // The code managed to establish an automapping for this filter option. Type = MappingType.AutoMapped, @@ -1080,7 +972,7 @@ public async Task PartiallyComplete() }, { "filter-1-option-2-key", - mappings.GetFilterOptionMapping("filter-1-key", "filter-1-option-2-key") with + mapping.GetFilterOptionMapping("filter-1-key", "filter-1-option-2-key") with { // The code managed to establish that no obvious automapping candidate exists for // this filter option. @@ -1092,7 +984,7 @@ public async Task PartiallyComplete() } }, { - "filter-2-key", mappings.GetFilterMapping("filter-2-key") with + "filter-2-key", mapping.GetFilterMapping("filter-2-key") with { // The code managed to establish that no obvious automapping candidate exists for // this filter. @@ -1102,7 +994,7 @@ public async Task PartiallyComplete() { { "filter-2-option-1-key", - mappings.GetFilterOptionMapping("filter-2-key", "filter-2-option-1-key") with + mapping.GetFilterOptionMapping("filter-2-key", "filter-2-option-1-key") with { // The code managed to establish that no obvious automapping candidate exists for // this filter option. @@ -1115,21 +1007,15 @@ public async Task PartiallyComplete() } }; - updatedMappings.FilterMappingPlan.Mappings.AssertDeepEqualTo( + updatedMapping.FilterMappingPlan.Mappings.AssertDeepEqualTo( expectedFilterMappings, ignoreCollectionOrders: true); - Assert.False(updatedMappings.FilterMappingsComplete); + Assert.False(updatedMapping.FilterMappingsComplete); // Some source filter options have no equivalent candidate to be mapped to, thus // resulting in a major version update. - Assert.Equal("2.0.0", updatedMappings.TargetDataSetVersion.SemVersion()); - - var updatedReleaseFile = await GetDbContext() - .ReleaseFiles - .SingleAsync(rf => rf.PublicApiDataSetId == updatedMappings.TargetDataSetVersion.DataSetId); - - Assert.Equal(updatedMappings.TargetDataSetVersion.SemVersion(), updatedReleaseFile.PublicApiDataSetVersion); + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); } [Fact] @@ -1140,7 +1026,7 @@ public async Task Complete_ExactMatch() // Create a mapping plan based on 2 data set versions with exactly the same filters // and filter options. - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() .WithSourceDataSetVersionId(originalVersion.Id) .WithTargetDataSetVersionId(nextVersion.Id) @@ -1172,25 +1058,16 @@ public async Task Complete_ExactMatch() .AddOptionCandidate("filter-2-option-1-key", DataFixture .DefaultMappableFilterOption()))); - await AddTestData(context => context.DataSetVersionMappings.Add(mappings)); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile()) - .WithPublicApiDataSetId(nextVersion.DataSetId) - .WithPublicApiDataSetVersion(nextVersion.SemVersion()); - - await AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); await ApplyAutoMappings(instanceId); - var updatedMappings = GetDataSetVersionMapping(nextVersion); + var updatedMapping = await GetDataSetVersionMapping(nextVersion); Dictionary expectedFilterMappings = new() { { - "filter-1-key", mappings.GetFilterMapping("filter-1-key") with + "filter-1-key", mapping.GetFilterMapping("filter-1-key") with { // The code managed to establish an automapping for this filter. Type = MappingType.AutoMapped, @@ -1199,7 +1076,7 @@ public async Task Complete_ExactMatch() { { "filter-1-option-1-key", - mappings.GetFilterOptionMapping("filter-1-key", "filter-1-option-1-key") with + mapping.GetFilterOptionMapping("filter-1-key", "filter-1-option-1-key") with { // The code managed to establish an automapping for this filter option. Type = MappingType.AutoMapped, @@ -1208,7 +1085,7 @@ public async Task Complete_ExactMatch() }, { "filter-1-option-2-key", - mappings.GetFilterOptionMapping("filter-1-key", "filter-1-option-2-key") with + mapping.GetFilterOptionMapping("filter-1-key", "filter-1-option-2-key") with { // The code managed to establish an automapping for this filter option. Type = MappingType.AutoMapped, @@ -1219,7 +1096,7 @@ public async Task Complete_ExactMatch() } }, { - "filter-2-key", mappings.GetFilterMapping("filter-2-key") with + "filter-2-key", mapping.GetFilterMapping("filter-2-key") with { // The code managed to establish an automapping for this filter. Type = MappingType.AutoMapped, @@ -1228,7 +1105,7 @@ public async Task Complete_ExactMatch() { { "filter-2-option-1-key", - mappings.GetFilterOptionMapping("filter-2-key", "filter-2-option-1-key") with + mapping.GetFilterOptionMapping("filter-2-key", "filter-2-option-1-key") with { // The code managed to establish an automapping for this filter option. Type = MappingType.AutoMapped, @@ -1240,19 +1117,13 @@ public async Task Complete_ExactMatch() } }; - updatedMappings.FilterMappingPlan.Mappings.AssertDeepEqualTo(expectedFilterMappings); + updatedMapping.FilterMappingPlan.Mappings.AssertDeepEqualTo(expectedFilterMappings); - Assert.True(updatedMappings.FilterMappingsComplete); + Assert.True(updatedMapping.FilterMappingsComplete); // All source filter options have equivalent candidates to be mapped to, thus // resulting in a minor version update. - Assert.Equal("1.1.0", updatedMappings.TargetDataSetVersion.SemVersion()); - - var updatedReleaseFile = await GetDbContext() - .ReleaseFiles - .SingleAsync(rf => rf.PublicApiDataSetId == updatedMappings.TargetDataSetVersion.DataSetId); - - Assert.Equal(updatedMappings.TargetDataSetVersion.SemVersion(), updatedReleaseFile.PublicApiDataSetVersion); + await AssertCorrectDataSetVersionNumbers(updatedMapping, "1.1.0"); } [Fact] @@ -1266,7 +1137,7 @@ public async Task Complete_AllSourcesMapped_OtherUnmappedCandidatesExist() // Each source filter and filter option can be auto-mapped exactly to one in // the target version, leaving some candidates unused but essentially the mapping // is complete unless the user manually intervenes at this point. - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() .WithSourceDataSetVersionId(originalVersion.Id) .WithTargetDataSetVersionId(nextVersion.Id) @@ -1289,25 +1160,16 @@ public async Task Complete_AllSourcesMapped_OtherUnmappedCandidatesExist() .AddOptionCandidate("filter-2-option-1-key", DataFixture .DefaultMappableFilterOption()))); - await AddTestData(context => context.DataSetVersionMappings.Add(mappings)); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile()) - .WithPublicApiDataSetId(nextVersion.DataSetId) - .WithPublicApiDataSetVersion(nextVersion.SemVersion()); - - await AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); await ApplyAutoMappings(instanceId); - var updatedMappings = GetDataSetVersionMapping(nextVersion); + var updatedMapping = await GetDataSetVersionMapping(nextVersion); Dictionary expectedFilterMappings = new() { { - "filter-1-key", mappings.GetFilterMapping("filter-1-key") with + "filter-1-key", mapping.GetFilterMapping("filter-1-key") with { Type = MappingType.AutoMapped, CandidateKey = "filter-1-key", @@ -1315,7 +1177,7 @@ public async Task Complete_AllSourcesMapped_OtherUnmappedCandidatesExist() { { "filter-1-option-1-key", - mappings.GetFilterOptionMapping("filter-1-key", "filter-1-option-1-key") with + mapping.GetFilterOptionMapping("filter-1-key", "filter-1-option-1-key") with { Type = MappingType.AutoMapped, CandidateKey = "filter-1-option-1-key" @@ -1326,22 +1188,16 @@ public async Task Complete_AllSourcesMapped_OtherUnmappedCandidatesExist() } }; - updatedMappings.FilterMappingPlan.Mappings.AssertDeepEqualTo( + updatedMapping.FilterMappingPlan.Mappings.AssertDeepEqualTo( expectedFilterMappings, ignoreCollectionOrders: true); - Assert.True(updatedMappings.FilterMappingsComplete); + Assert.True(updatedMapping.FilterMappingsComplete); // All source filter options have equivalent candidates to be mapped to, thus // resulting in a minor version update. The inclusion of new filter options // not present in the original version does not matter. - Assert.Equal("1.1.0", updatedMappings.TargetDataSetVersion.SemVersion()); - - var updatedReleaseFile = await GetDbContext() - .ReleaseFiles - .SingleAsync(rf => rf.PublicApiDataSetId == updatedMappings.TargetDataSetVersion.DataSetId); - - Assert.Equal(updatedMappings.TargetDataSetVersion.SemVersion(), updatedReleaseFile.PublicApiDataSetVersion); + await AssertCorrectDataSetVersionNumbers(updatedMapping, "1.1.0"); } // As there is currently no way in the UI for a user to resolve unmapped filters, filters @@ -1358,7 +1214,7 @@ public async Task Complete_SomeFiltersAutoNone() // Each source filter and filter option can be auto-mapped exactly to one in // the target version, leaving some candidates unused but essentially the mapping // is complete unless the user manually intervenes at this point. - DataSetVersionMapping mappings = DataFixture + DataSetVersionMapping mapping = DataFixture .DefaultDataSetVersionMapping() .WithSourceDataSetVersionId(originalVersion.Id) .WithTargetDataSetVersionId(nextVersion.Id) @@ -1381,25 +1237,16 @@ public async Task Complete_SomeFiltersAutoNone() .AddOptionCandidate("filter-1-option-1-key", DataFixture .DefaultMappableFilterOption()))); - await AddTestData(context => context.DataSetVersionMappings.Add(mappings)); - - ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() - .WithId(nextVersion.Release.ReleaseFileId) - .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) - .WithFile(DataFixture.DefaultFile()) - .WithPublicApiDataSetId(nextVersion.DataSetId) - .WithPublicApiDataSetVersion(nextVersion.SemVersion()); - - await AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); await ApplyAutoMappings(instanceId); - var updatedMappings = GetDataSetVersionMapping(nextVersion); + var updatedMapping = await GetDataSetVersionMapping(nextVersion); Dictionary expectedFilterMappings = new() { { - "filter-1-key", mappings.GetFilterMapping("filter-1-key") with + "filter-1-key", mapping.GetFilterMapping("filter-1-key") with { Type = MappingType.AutoMapped, CandidateKey = "filter-1-key", @@ -1407,7 +1254,7 @@ public async Task Complete_SomeFiltersAutoNone() { { "filter-1-option-1-key", - mappings.GetFilterOptionMapping("filter-1-key", "filter-1-option-1-key") with + mapping.GetFilterOptionMapping("filter-1-key", "filter-1-option-1-key") with { Type = MappingType.AutoMapped, CandidateKey = "filter-1-option-1-key" @@ -1417,14 +1264,14 @@ public async Task Complete_SomeFiltersAutoNone() } }, { - "filter-2-key", mappings.GetFilterMapping("filter-2-key") with + "filter-2-key", mapping.GetFilterMapping("filter-2-key") with { Type = MappingType.AutoNone, OptionMappings = new Dictionary { { "filter-2-option-1-key", - mappings.GetFilterOptionMapping("filter-2-key", "filter-2-option-1-key") with + mapping.GetFilterOptionMapping("filter-2-key", "filter-2-option-1-key") with { Type = MappingType.AutoNone } @@ -1434,21 +1281,15 @@ public async Task Complete_SomeFiltersAutoNone() } }; - updatedMappings.FilterMappingPlan.Mappings.AssertDeepEqualTo( + updatedMapping.FilterMappingPlan.Mappings.AssertDeepEqualTo( expectedFilterMappings, ignoreCollectionOrders: true); - Assert.True(updatedMappings.FilterMappingsComplete); + Assert.True(updatedMapping.FilterMappingsComplete); // Some source filter options have no equivalent candidate to be mapped to, thus // resulting in a major version update. - Assert.Equal("2.0.0", updatedMappings.TargetDataSetVersion.SemVersion()); - - var updatedReleaseFile = await GetDbContext() - .ReleaseFiles - .SingleAsync(rf => rf.PublicApiDataSetId == updatedMappings.TargetDataSetVersion.DataSetId); - - Assert.Equal(updatedMappings.TargetDataSetVersion.SemVersion(), updatedReleaseFile.PublicApiDataSetVersion); + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); } } @@ -1496,21 +1337,42 @@ await CreateDataSetInitialAndNextVersion( nextVersionStatus: DataSetVersionStatus.Processing); SetupCsvDataFilesForDataSetVersion(ProcessorTestData.AbsenceSchool, nextDataSetVersion); + + ReleaseFile releaseFile = DataFixture.DefaultReleaseFile() + .WithId(nextDataSetVersion.Release.ReleaseFileId) + .WithReleaseVersion(DataFixture.DefaultReleaseVersion()) + .WithFile(DataFixture.DefaultFile()) + .WithPublicApiDataSetId(nextDataSetVersion.DataSetId) + .WithPublicApiDataSetVersion(nextDataSetVersion.SemVersion()); + + await AddTestData(context => context.ReleaseFiles.Add(releaseFile)); + return (instanceId, initialDataSetVersion, nextDataSetVersion); } - private DataSetVersion GetDataSetVersion(DataSetVersion nextVersion) + private async Task GetDataSetVersion(DataSetVersion nextVersion) { - return GetDbContext() + return await GetDbContext() .DataSetVersions - .Single(dsv => dsv.Id == nextVersion.Id); + .SingleAsync(dsv => dsv.Id == nextVersion.Id); } - private DataSetVersionMapping GetDataSetVersionMapping(DataSetVersion nextVersion) + private async Task GetDataSetVersionMapping(DataSetVersion nextVersion) { - return GetDbContext() + return await GetDbContext() .DataSetVersionMappings .Include(mapping => mapping.TargetDataSetVersion) - .Single(mapping => mapping.TargetDataSetVersionId == nextVersion.Id); + .SingleAsync(mapping => mapping.TargetDataSetVersionId == nextVersion.Id); + } + + private async Task AssertCorrectDataSetVersionNumbers(DataSetVersionMapping mapping, string expectedVersion) + { + Assert.Equal(expectedVersion, mapping.TargetDataSetVersion.SemVersion().ToString()); + + var updatedReleaseFile = await GetDbContext() + .ReleaseFiles + .SingleAsync(rf => rf.PublicApiDataSetId == mapping.TargetDataSetVersion.DataSetId); + + Assert.Equal(expectedVersion, updatedReleaseFile.PublicApiDataSetVersion?.ToString()); } } From 47bc7a4048404be571a39ec3c30d67472dcc2b9e Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Thu, 10 Oct 2024 18:14:52 +0100 Subject: [PATCH 61/80] EES-5563 Fix `FilterMetaRepository.ReadFilterMetas` updating sequences This fixes the `ReadFilterMetas` method updating the meta ID sequence by accident when it should be read-only. --- .../Repository/FilterMetaRepository.cs | 69 +++++++++---------- 1 file changed, 32 insertions(+), 37 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterMetaRepository.cs index 89a29772b64..fb25f3163d4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterMetaRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/FilterMetaRepository.cs @@ -32,7 +32,6 @@ public async Task>> ReadFilterMet duckDbConnection, dataSetVersion, allowedColumns, - publicIdMappings: [], cancellationToken); return await metas @@ -56,16 +55,36 @@ public async Task CreateFilterMetas( { var publicIdMappings = await CreatePublicIdMappings(dataSetVersion, cancellationToken); + var currentMetaId = await publicDataDbContext.NextSequenceValue( + PublicDataDbContext.FilterMetasIdSequence, + cancellationToken); + var metas = await GetFilterMetas( duckDbConnection, dataSetVersion, allowedColumns, - publicIdMappings.Filters, cancellationToken); + foreach (var meta in metas) + { + meta.Id = currentMetaId++; + meta.PublicId = publicIdMappings.Filters.GetValueOrDefault(meta.Column, SqidEncoder.Encode(meta.Id)); + } + publicDataDbContext.FilterMetas.AddRange(metas); await publicDataDbContext.SaveChangesAsync(cancellationToken); + // Avoid trying to set to 0 (which only + // happens synthetically during tests). + if (currentMetaId > 1) + { + await publicDataDbContext.SetSequenceValue( + PublicDataDbContext.FilterMetasIdSequence, + currentMetaId - 1, + cancellationToken + ); + } + foreach (var meta in metas) { var options = await GetFilterOptionMeta( @@ -88,7 +107,7 @@ await optionTable .InsertWhenNotMatched() .MergeAsync(cancellationToken); - var currentId = await publicDataDbContext.NextSequenceValue( + var currentLinkId = await publicDataDbContext.NextSequenceValue( PublicDataDbContext.FilterOptionMetaLinkSequence, cancellationToken); @@ -121,7 +140,7 @@ await optionTable publicIdMappings: publicIdMappings, filter: meta, option: option, - defaultPublicIdFn: () => SqidEncoder.Encode(currentId++)), + defaultPublicIdFn: () => SqidEncoder.Encode(currentLinkId++)), MetaId = meta.Id, OptionId = option.Id }) @@ -146,12 +165,12 @@ await optionTable // Avoid trying to set to 0 (which only // happens synthetically during tests). - if (currentId > 1) + if (currentLinkId > 1) { // Increase the sequence only by the amount that we used to generate new PublicIds. await publicDataDbContext.SetSequenceValue( PublicDataDbContext.FilterOptionMetaLinkSequence, - currentId - 1, + currentLinkId - 1, cancellationToken ); } @@ -162,13 +181,8 @@ private async Task> GetFilterMetas( IDuckDbConnection duckDbConnection, DataSetVersion dataSetVersion, IReadOnlySet allowedColumns, - Dictionary publicIdMappings, CancellationToken cancellationToken) { - var currentId = await publicDataDbContext.NextSequenceValue( - PublicDataDbContext.FilterMetasIdSequence, - cancellationToken); - var metaRows = await duckDbConnection.SqlBuilder( $""" SELECT * @@ -178,36 +192,17 @@ private async Task> GetFilterMetas( """) .QueryAsync(cancellationToken: cancellationToken); - var metas = metaRows + return metaRows .OrderBy(row => row.Label) - .Select(row => + .Select(row => new FilterMeta { - var id = currentId++; - - return new FilterMeta - { - Id = id, - PublicId = publicIdMappings.GetValueOrDefault(row.ColName, SqidEncoder.Encode(id)), - Column = row.ColName, - DataSetVersionId = dataSetVersion.Id, - Label = row.Label, - Hint = row.FilterHint ?? string.Empty - }; + PublicId = string.Empty, + Column = row.ColName, + DataSetVersionId = dataSetVersion.Id, + Label = row.Label, + Hint = row.FilterHint ?? string.Empty }) .ToList(); - - // Avoid trying to set to 0 (which only - // happens synthetically during tests). - if (currentId > 1) - { - await publicDataDbContext.SetSequenceValue( - PublicDataDbContext.FilterMetasIdSequence, - currentId - 1, - cancellationToken - ); - } - - return metas; } private async Task> GetFilterOptionMeta( From e7b7b024fcacdc3b42a39fa8f6b7550222adc20d Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Fri, 11 Oct 2024 12:18:46 +0100 Subject: [PATCH 62/80] EES-5563 Add meta flags to `DataSetVersionMapping` for version calculations --- ...ataSetVersionMappingGeneratorExtensions.cs | 36 +- .../DataSetVersionMapping.cs | 11 + ...taFlagsToDataSetVersionMapping.Designer.cs | 1460 +++++++++++++++++ ...DeletedMetaFlagsToDataSetVersionMapping.cs | 51 + .../PublicDataDbContextModelSnapshot.cs | 9 + ...NextDataSetVersionMappingsFunctionTests.cs | 199 ++- .../Model/DataSertVersionMappingMeta.cs | 8 - .../Models/DataSetVersionMappingMeta.cs | 18 + .../GeographicLevelMetaRepository.cs | 3 +- .../Repository/IndicatorMetaRepository.cs | 72 +- .../Interfaces/IIndicatorMetaRepository.cs | 6 + .../Services/DataSetMetaService.cs | 19 +- .../Services/DataSetVersionMappingService.cs | 70 +- .../Interfaces/IDataSetMetaService.cs | 4 +- 14 files changed, 1916 insertions(+), 50 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/20241013014815_EES5563_AddDeletedMetaFlagsToDataSetVersionMapping.Designer.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/20241013014815_EES5563_AddDeletedMetaFlagsToDataSetVersionMapping.cs delete mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/DataSertVersionMappingMeta.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Models/DataSetVersionMappingMeta.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/DataSetVersionMappingGeneratorExtensions.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/DataSetVersionMappingGeneratorExtensions.cs index a2a5929444b..042f31774a2 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/DataSetVersionMappingGeneratorExtensions.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests/Fixtures/DataSetVersionMappingGeneratorExtensions.cs @@ -40,6 +40,21 @@ public static Generator WithFilterMappingPlan( FilterMappingPlan filterMappingPlan) => generator.ForInstance(s => s.SetFilterMappingPlan(filterMappingPlan)); + public static Generator WithHasDeletedIndicators( + this Generator generator, + bool hasDeletedIndicators) + => generator.ForInstance(s => s.SetHasDeletedIndicators(hasDeletedIndicators)); + + public static Generator WithHasDeletedGeographicLevels( + this Generator generator, + bool hasDeletedGeographicLevels) + => generator.ForInstance(s => s.SetHasDeletedGeographicLevels(hasDeletedGeographicLevels)); + + public static Generator WithHasDeletedTimePeriods( + this Generator generator, + bool hasDeletedTimePeriods) + => generator.ForInstance(s => s.SetHasDeletedTimePeriods(hasDeletedTimePeriods)); + public static InstanceSetters SetDefaults( this InstanceSetters setters) => setters @@ -76,12 +91,25 @@ public static InstanceSetters SetTargetDataSetVersionId( public static InstanceSetters SetLocationMappingPlan( this InstanceSetters instanceSetter, LocationMappingPlan locationMappingPlan) - => instanceSetter - .Set(mapping => mapping.LocationMappingPlan, locationMappingPlan); + => instanceSetter.Set(mapping => mapping.LocationMappingPlan, locationMappingPlan); public static InstanceSetters SetFilterMappingPlan( this InstanceSetters instanceSetter, FilterMappingPlan filterMappingPlan) - => instanceSetter - .Set(mapping => mapping.FilterMappingPlan, filterMappingPlan); + => instanceSetter.Set(mapping => mapping.FilterMappingPlan, filterMappingPlan); + + public static InstanceSetters SetHasDeletedIndicators( + this InstanceSetters instanceSetter, + bool hasDeletedIndicators) + => instanceSetter.Set(mapping => mapping.HasDeletedIndicators, hasDeletedIndicators); + + public static InstanceSetters SetHasDeletedGeographicLevels( + this InstanceSetters instanceSetter, + bool hasDeletedGeographicLevels) + => instanceSetter.Set(mapping => mapping.HasDeletedGeographicLevels, hasDeletedGeographicLevels); + + public static InstanceSetters SetHasDeletedTimePeriods( + this InstanceSetters instanceSetter, + bool hasDeletedTimePeriods) + => instanceSetter.Set(mapping => mapping.HasDeletedTimePeriods, hasDeletedTimePeriods); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionMapping.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionMapping.cs index dc50b957815..2d55ac1af31 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionMapping.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/DataSetVersionMapping.cs @@ -28,6 +28,17 @@ public class DataSetVersionMapping : ICreatedUpdatedTimestamps +using System; +using System.Collections.Generic; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Migrations +{ + [DbContext(typeof(PublicDataDbContext))] + [Migration("20241013014815_EES5563_AddDeletedMetaFlagsToDataSetVersionMapping")] + partial class EES5563_AddDeletedMetaFlagsToDataSetVersionMapping + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.HasSequence("FilterOptionMetaLink_seq"); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("LatestDraftVersionId") + .HasColumnType("uuid"); + + b.Property("LatestLiveVersionId") + .HasColumnType("uuid"); + + b.Property("PublicationId") + .HasColumnType("uuid"); + + b.Property("Published") + .HasColumnType("timestamp with time zone"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text"); + + b.Property("Summary") + .IsRequired() + .HasColumnType("text"); + + b.Property("SupersedingDataSetId") + .HasColumnType("uuid"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.Property("Withdrawn") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("LatestDraftVersionId") + .IsUnique(); + + b.HasIndex("LatestLiveVersionId") + .IsUnique(); + + b.HasIndex("SupersedingDataSetId"); + + b.ToTable("DataSets"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetId") + .HasColumnType("uuid"); + + b.Property("Notes") + .IsRequired() + .HasColumnType("text"); + + b.Property("Published") + .HasColumnType("timestamp with time zone"); + + b.Property("Status") + .IsRequired() + .HasColumnType("text"); + + b.Property("TotalResults") + .HasColumnType("bigint"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.Property("VersionMajor") + .HasColumnType("integer"); + + b.Property("VersionMinor") + .HasColumnType("integer"); + + b.Property("VersionPatch") + .HasColumnType("integer"); + + b.Property("Withdrawn") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetId", "VersionMajor", "VersionMinor", "VersionPatch") + .IsUnique() + .HasDatabaseName("IX_DataSetVersions_DataSetId_VersionNumber"); + + b.ToTable("DataSetVersions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersionImport", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Completed") + .HasColumnType("timestamp with time zone"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("InstanceId") + .HasColumnType("uuid"); + + b.Property("Stage") + .IsRequired() + .HasColumnType("text"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId"); + + b.HasIndex("InstanceId") + .IsUnique(); + + b.ToTable("DataSetVersionImports"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersionMapping", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("FilterMappingPlan") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("FilterMappingsComplete") + .HasColumnType("boolean"); + + b.Property("HasDeletedGeographicLevels") + .HasColumnType("boolean"); + + b.Property("HasDeletedIndicators") + .HasColumnType("boolean"); + + b.Property("HasDeletedTimePeriods") + .HasColumnType("boolean"); + + b.Property("LocationMappingPlan") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("LocationMappingsComplete") + .HasColumnType("boolean"); + + b.Property("SourceDataSetVersionId") + .HasColumnType("uuid"); + + b.Property("TargetDataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("SourceDataSetVersionId") + .IsUnique() + .HasDatabaseName("IX_DataSetVersionMappings_SourceDataSetVersionId"); + + b.HasIndex("TargetDataSetVersionId") + .IsUnique() + .HasDatabaseName("IX_DataSetVersionMappings_TargetDataSetVersionId"); + + b.ToTable("DataSetVersionMappings"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Column") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Hint") + .IsRequired() + .HasColumnType("text"); + + b.Property("Label") + .IsRequired() + .HasMaxLength(80) + .HasColumnType("character varying(80)"); + + b.Property("PublicId") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("character varying(10)"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId", "Column") + .IsUnique(); + + b.HasIndex("DataSetVersionId", "PublicId") + .IsUnique(); + + b.ToTable("FilterMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMetaChange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CurrentStateId") + .HasColumnType("integer"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("PreviousStateId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("CurrentStateId"); + + b.HasIndex("DataSetVersionId"); + + b.HasIndex("PreviousStateId"); + + b.ToTable("FilterMetaChanges"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("IsAggregate") + .HasColumnType("boolean"); + + b.Property("Label") + .IsRequired() + .HasMaxLength(120) + .HasColumnType("character varying(120)"); + + b.HasKey("Id"); + + b.ToTable("FilterOptionMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMetaChange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId"); + + b.ToTable("FilterOptionMetaChanges"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMetaLink", b => + { + b.Property("MetaId") + .HasColumnType("integer"); + + b.Property("OptionId") + .HasColumnType("integer"); + + b.Property("PublicId") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("character varying(10)"); + + b.HasKey("MetaId", "OptionId"); + + b.HasIndex("OptionId"); + + b.HasIndex("MetaId", "PublicId") + .IsUnique(); + + b.ToTable("FilterOptionMetaLinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.GeographicLevelMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property>("Levels") + .IsRequired() + .HasColumnType("text[]"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId") + .IsUnique(); + + b.ToTable("GeographicLevelMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.GeographicLevelMetaChange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CurrentStateId") + .HasColumnType("integer"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("PreviousStateId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("CurrentStateId"); + + b.HasIndex("DataSetVersionId") + .IsUnique(); + + b.HasIndex("PreviousStateId"); + + b.ToTable("GeographicLevelMetaChanges"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.IndicatorMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Column") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("DecimalPlaces") + .HasColumnType("smallint"); + + b.Property("Label") + .IsRequired() + .HasMaxLength(80) + .HasColumnType("character varying(80)"); + + b.Property("PublicId") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("character varying(10)"); + + b.Property("Unit") + .HasColumnType("text"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId", "Column") + .IsUnique(); + + b.HasIndex("DataSetVersionId", "PublicId") + .IsUnique(); + + b.ToTable("IndicatorMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.IndicatorMetaChange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CurrentStateId") + .HasColumnType("integer"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("PreviousStateId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("CurrentStateId"); + + b.HasIndex("DataSetVersionId"); + + b.HasIndex("PreviousStateId"); + + b.ToTable("IndicatorMetaChanges"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Level") + .IsRequired() + .HasMaxLength(5) + .HasColumnType("character varying(5)"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId", "Level") + .IsUnique(); + + b.ToTable("LocationMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMetaChange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CurrentStateId") + .HasColumnType("integer"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("PreviousStateId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("CurrentStateId"); + + b.HasIndex("DataSetVersionId"); + + b.HasIndex("PreviousStateId"); + + b.ToTable("LocationMetaChanges"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Code") + .HasMaxLength(30) + .HasColumnType("character varying(30)"); + + b.Property("LaEstab") + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("Label") + .IsRequired() + .HasMaxLength(120) + .HasColumnType("character varying(120)"); + + b.Property("OldCode") + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("character varying(10)"); + + b.Property("Ukprn") + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("Urn") + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.HasKey("Id"); + + b.HasIndex("Code"); + + b.HasIndex("LaEstab"); + + b.HasIndex("OldCode"); + + b.HasIndex("Type"); + + b.HasIndex("Ukprn"); + + b.HasIndex("Urn"); + + b.HasIndex(new[] { "Type", "Label", "Code", "OldCode", "Urn", "LaEstab", "Ukprn" }, "IX_LocationOptionMetas_All") + .IsUnique(); + + NpgsqlIndexBuilderExtensions.AreNullsDistinct(b.HasIndex(new[] { "Type", "Label", "Code", "OldCode", "Urn", "LaEstab", "Ukprn" }, "IX_LocationOptionMetas_All"), false); + + b.ToTable("LocationOptionMetas"); + + b.HasDiscriminator("Type").HasValue("LocationOptionMeta"); + + b.UseTphMappingStrategy(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMetaChange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId"); + + b.ToTable("LocationOptionMetaChanges"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMetaLink", b => + { + b.Property("MetaId") + .HasColumnType("integer"); + + b.Property("OptionId") + .HasColumnType("integer"); + + b.Property("PublicId") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("character varying(10)"); + + b.HasKey("MetaId", "OptionId"); + + b.HasIndex("OptionId"); + + b.HasIndex("MetaId", "PublicId") + .IsUnique(); + + b.ToTable("LocationOptionMetaLinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.PreviewToken", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("CreatedByUserId") + .HasColumnType("uuid"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Expiry") + .HasColumnType("timestamp with time zone"); + + b.Property("Label") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId"); + + b.ToTable("PreviewTokens"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodMeta", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Code") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("character varying(10)"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("Period") + .IsRequired() + .HasMaxLength(9) + .HasColumnType("character varying(9)"); + + b.Property("Updated") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("DataSetVersionId", "Code", "Period") + .IsUnique(); + + b.ToTable("TimePeriodMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodMetaChange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CurrentStateId") + .HasColumnType("integer"); + + b.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b.Property("PreviousStateId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("CurrentStateId"); + + b.HasIndex("DataSetVersionId"); + + b.HasIndex("PreviousStateId"); + + b.ToTable("TimePeriodMetaChanges"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationCodedOptionMeta", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta"); + + b.HasDiscriminator().HasValue("CODE"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationLocalAuthorityOptionMeta", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta"); + + b.HasDiscriminator().HasValue("LA"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationProviderOptionMeta", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta"); + + b.HasDiscriminator().HasValue("PROV"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationRscRegionOptionMeta", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta"); + + b.HasDiscriminator().HasValue("RSC"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationSchoolOptionMeta", b => + { + b.HasBaseType("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta"); + + b.HasDiscriminator().HasValue("SCH"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "LatestDraftVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", "LatestDraftVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "LatestLiveVersion") + .WithOne() + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", "LatestLiveVersionId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", "SupersedingDataSet") + .WithMany() + .HasForeignKey("SupersedingDataSetId"); + + b.Navigation("LatestDraftVersion"); + + b.Navigation("LatestLiveVersion"); + + b.Navigation("SupersedingDataSet"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", "DataSet") + .WithMany("Versions") + .HasForeignKey("DataSetId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersionMetaSummary", "MetaSummary", b1 => + { + b1.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b1.Property>("Filters") + .IsRequired() + .HasColumnType("text[]"); + + b1.Property>("GeographicLevels") + .IsRequired() + .HasColumnType("text[]"); + + b1.Property>("Indicators") + .IsRequired() + .HasColumnType("text[]"); + + b1.HasKey("DataSetVersionId"); + + b1.ToTable("DataSetVersions"); + + b1.ToJson("MetaSummary"); + + b1.WithOwner() + .HasForeignKey("DataSetVersionId"); + + b1.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodRange", "TimePeriodRange", b2 => + { + b2.Property("DataSetVersionMetaSummaryDataSetVersionId") + .HasColumnType("uuid"); + + b2.HasKey("DataSetVersionMetaSummaryDataSetVersionId"); + + b2.ToTable("DataSetVersions"); + + b2.WithOwner() + .HasForeignKey("DataSetVersionMetaSummaryDataSetVersionId"); + + b2.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodRangeBound", "End", b3 => + { + b3.Property("TimePeriodRangeDataSetVersionMetaSummaryDataSetVersionId") + .HasColumnType("uuid"); + + b3.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b3.Property("Period") + .IsRequired() + .HasColumnType("text"); + + b3.HasKey("TimePeriodRangeDataSetVersionMetaSummaryDataSetVersionId"); + + b3.ToTable("DataSetVersions"); + + b3.WithOwner() + .HasForeignKey("TimePeriodRangeDataSetVersionMetaSummaryDataSetVersionId"); + }); + + b2.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodRangeBound", "Start", b3 => + { + b3.Property("TimePeriodRangeDataSetVersionMetaSummaryDataSetVersionId") + .HasColumnType("uuid"); + + b3.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b3.Property("Period") + .IsRequired() + .HasColumnType("text"); + + b3.HasKey("TimePeriodRangeDataSetVersionMetaSummaryDataSetVersionId"); + + b3.ToTable("DataSetVersions"); + + b3.WithOwner() + .HasForeignKey("TimePeriodRangeDataSetVersionMetaSummaryDataSetVersionId"); + }); + + b2.Navigation("End") + .IsRequired(); + + b2.Navigation("Start") + .IsRequired(); + }); + + b1.Navigation("TimePeriodRange") + .IsRequired(); + }); + + b.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Release", "Release", b1 => + { + b1.Property("DataSetVersionId") + .HasColumnType("uuid"); + + b1.Property("DataSetFileId") + .HasColumnType("uuid"); + + b1.Property("ReleaseFileId") + .HasColumnType("uuid"); + + b1.Property("Slug") + .IsRequired() + .HasColumnType("text"); + + b1.Property("Title") + .IsRequired() + .HasColumnType("text"); + + b1.HasKey("DataSetVersionId"); + + b1.HasIndex("DataSetFileId"); + + b1.HasIndex("ReleaseFileId") + .IsUnique(); + + b1.ToTable("DataSetVersions"); + + b1.WithOwner() + .HasForeignKey("DataSetVersionId"); + }); + + b.Navigation("DataSet"); + + b.Navigation("MetaSummary"); + + b.Navigation("Release") + .IsRequired(); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersionImport", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("Imports") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersionMapping", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "SourceDataSetVersion") + .WithMany() + .HasForeignKey("SourceDataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "TargetDataSetVersion") + .WithMany() + .HasForeignKey("TargetDataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("SourceDataSetVersion"); + + b.Navigation("TargetDataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("FilterMetas") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMetaChange", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta", "CurrentState") + .WithMany() + .HasForeignKey("CurrentStateId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("FilterMetaChanges") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta", "PreviousState") + .WithMany() + .HasForeignKey("PreviousStateId"); + + b.Navigation("CurrentState"); + + b.Navigation("DataSetVersion"); + + b.Navigation("PreviousState"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMetaChange", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("FilterOptionMetaChanges") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMetaChange+State", "CurrentState", b1 => + { + b1.Property("FilterOptionMetaChangeId") + .HasColumnType("bigint"); + + b1.Property("MetaId") + .HasColumnType("integer"); + + b1.Property("OptionId") + .HasColumnType("integer"); + + b1.Property("PublicId") + .IsRequired() + .HasColumnType("text"); + + b1.HasKey("FilterOptionMetaChangeId"); + + b1.HasIndex("MetaId"); + + b1.HasIndex("OptionId"); + + b1.ToTable("FilterOptionMetaChanges"); + + b1.WithOwner() + .HasForeignKey("FilterOptionMetaChangeId"); + + b1.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta", "Meta") + .WithMany() + .HasForeignKey("MetaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b1.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMeta", "Option") + .WithMany() + .HasForeignKey("OptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b1.Navigation("Meta"); + + b1.Navigation("Option"); + }); + + b.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMetaChange+State", "PreviousState", b1 => + { + b1.Property("FilterOptionMetaChangeId") + .HasColumnType("bigint"); + + b1.Property("MetaId") + .HasColumnType("integer"); + + b1.Property("OptionId") + .HasColumnType("integer"); + + b1.Property("PublicId") + .IsRequired() + .HasColumnType("text"); + + b1.HasKey("FilterOptionMetaChangeId"); + + b1.HasIndex("MetaId"); + + b1.HasIndex("OptionId"); + + b1.ToTable("FilterOptionMetaChanges"); + + b1.WithOwner() + .HasForeignKey("FilterOptionMetaChangeId"); + + b1.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta", "Meta") + .WithMany() + .HasForeignKey("MetaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b1.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMeta", "Option") + .WithMany() + .HasForeignKey("OptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b1.Navigation("Meta"); + + b1.Navigation("Option"); + }); + + b.Navigation("CurrentState"); + + b.Navigation("DataSetVersion"); + + b.Navigation("PreviousState"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMetaLink", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta", "Meta") + .WithMany("OptionLinks") + .HasForeignKey("MetaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMeta", "Option") + .WithMany("MetaLinks") + .HasForeignKey("OptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Meta"); + + b.Navigation("Option"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.GeographicLevelMeta", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithOne("GeographicLevelMeta") + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.GeographicLevelMeta", "DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.GeographicLevelMetaChange", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.GeographicLevelMeta", "CurrentState") + .WithMany() + .HasForeignKey("CurrentStateId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithOne("GeographicLevelMetaChange") + .HasForeignKey("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.GeographicLevelMetaChange", "DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.GeographicLevelMeta", "PreviousState") + .WithMany() + .HasForeignKey("PreviousStateId"); + + b.Navigation("CurrentState"); + + b.Navigation("DataSetVersion"); + + b.Navigation("PreviousState"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.IndicatorMeta", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("IndicatorMetas") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.IndicatorMetaChange", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.IndicatorMeta", "CurrentState") + .WithMany() + .HasForeignKey("CurrentStateId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("IndicatorMetaChanges") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.IndicatorMeta", "PreviousState") + .WithMany() + .HasForeignKey("PreviousStateId"); + + b.Navigation("CurrentState"); + + b.Navigation("DataSetVersion"); + + b.Navigation("PreviousState"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMeta", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("LocationMetas") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMetaChange", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMeta", "CurrentState") + .WithMany() + .HasForeignKey("CurrentStateId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("LocationMetaChanges") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMeta", "PreviousState") + .WithMany() + .HasForeignKey("PreviousStateId"); + + b.Navigation("CurrentState"); + + b.Navigation("DataSetVersion"); + + b.Navigation("PreviousState"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMetaChange", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("LocationOptionMetaChanges") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMetaChange+State", "CurrentState", b1 => + { + b1.Property("LocationOptionMetaChangeId") + .HasColumnType("bigint"); + + b1.Property("MetaId") + .HasColumnType("integer"); + + b1.Property("OptionId") + .HasColumnType("integer"); + + b1.Property("PublicId") + .IsRequired() + .HasColumnType("text"); + + b1.HasKey("LocationOptionMetaChangeId"); + + b1.HasIndex("MetaId"); + + b1.HasIndex("OptionId"); + + b1.ToTable("LocationOptionMetaChanges"); + + b1.WithOwner() + .HasForeignKey("LocationOptionMetaChangeId"); + + b1.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMeta", "Meta") + .WithMany() + .HasForeignKey("MetaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b1.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta", "Option") + .WithMany() + .HasForeignKey("OptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b1.Navigation("Meta"); + + b1.Navigation("Option"); + }); + + b.OwnsOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMetaChange+State", "PreviousState", b1 => + { + b1.Property("LocationOptionMetaChangeId") + .HasColumnType("bigint"); + + b1.Property("MetaId") + .HasColumnType("integer"); + + b1.Property("OptionId") + .HasColumnType("integer"); + + b1.Property("PublicId") + .IsRequired() + .HasColumnType("text"); + + b1.HasKey("LocationOptionMetaChangeId"); + + b1.HasIndex("MetaId"); + + b1.HasIndex("OptionId"); + + b1.ToTable("LocationOptionMetaChanges"); + + b1.WithOwner() + .HasForeignKey("LocationOptionMetaChangeId"); + + b1.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMeta", "Meta") + .WithMany() + .HasForeignKey("MetaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b1.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta", "Option") + .WithMany() + .HasForeignKey("OptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b1.Navigation("Meta"); + + b1.Navigation("Option"); + }); + + b.Navigation("CurrentState"); + + b.Navigation("DataSetVersion"); + + b.Navigation("PreviousState"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMetaLink", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMeta", "Meta") + .WithMany("OptionLinks") + .HasForeignKey("MetaId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta", "Option") + .WithMany("MetaLinks") + .HasForeignKey("OptionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Meta"); + + b.Navigation("Option"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.PreviewToken", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("PreviewTokens") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodMeta", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("TimePeriodMetas") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("DataSetVersion"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodMetaChange", b => + { + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodMeta", "CurrentState") + .WithMany() + .HasForeignKey("CurrentStateId"); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", "DataSetVersion") + .WithMany("TimePeriodMetaChanges") + .HasForeignKey("DataSetVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.TimePeriodMeta", "PreviousState") + .WithMany() + .HasForeignKey("PreviousStateId"); + + b.Navigation("CurrentState"); + + b.Navigation("DataSetVersion"); + + b.Navigation("PreviousState"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSet", b => + { + b.Navigation("Versions"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DataSetVersion", b => + { + b.Navigation("FilterMetaChanges"); + + b.Navigation("FilterMetas"); + + b.Navigation("FilterOptionMetaChanges"); + + b.Navigation("GeographicLevelMeta"); + + b.Navigation("GeographicLevelMetaChange"); + + b.Navigation("Imports"); + + b.Navigation("IndicatorMetaChanges"); + + b.Navigation("IndicatorMetas"); + + b.Navigation("LocationMetaChanges"); + + b.Navigation("LocationMetas"); + + b.Navigation("LocationOptionMetaChanges"); + + b.Navigation("PreviewTokens"); + + b.Navigation("TimePeriodMetaChanges"); + + b.Navigation("TimePeriodMetas"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta", b => + { + b.Navigation("OptionLinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterOptionMeta", b => + { + b.Navigation("MetaLinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationMeta", b => + { + b.Navigation("OptionLinks"); + }); + + modelBuilder.Entity("GovUk.Education.ExploreEducationStatistics.Public.Data.Model.LocationOptionMeta", b => + { + b.Navigation("MetaLinks"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/20241013014815_EES5563_AddDeletedMetaFlagsToDataSetVersionMapping.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/20241013014815_EES5563_AddDeletedMetaFlagsToDataSetVersionMapping.cs new file mode 100644 index 00000000000..4e3f84b14ad --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/20241013014815_EES5563_AddDeletedMetaFlagsToDataSetVersionMapping.cs @@ -0,0 +1,51 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Migrations +{ + /// + public partial class EES5563_AddDeletedMetaFlagsToDataSetVersionMapping : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "HasDeletedGeographicLevels", + table: "DataSetVersionMappings", + type: "boolean", + nullable: false, + defaultValue: false); + + migrationBuilder.AddColumn( + name: "HasDeletedIndicators", + table: "DataSetVersionMappings", + type: "boolean", + nullable: false, + defaultValue: false); + + migrationBuilder.AddColumn( + name: "HasDeletedTimePeriods", + table: "DataSetVersionMappings", + type: "boolean", + nullable: false, + defaultValue: false); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "HasDeletedGeographicLevels", + table: "DataSetVersionMappings"); + + migrationBuilder.DropColumn( + name: "HasDeletedIndicators", + table: "DataSetVersionMappings"); + + migrationBuilder.DropColumn( + name: "HasDeletedTimePeriods", + table: "DataSetVersionMappings"); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/PublicDataDbContextModelSnapshot.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/PublicDataDbContextModelSnapshot.cs index 99b0857a041..3ec06c68e24 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/PublicDataDbContextModelSnapshot.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Model/Migrations/PublicDataDbContextModelSnapshot.cs @@ -181,6 +181,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("FilterMappingsComplete") .HasColumnType("boolean"); + b.Property("HasDeletedGeographicLevels") + .HasColumnType("boolean"); + + b.Property("HasDeletedIndicators") + .HasColumnType("boolean"); + + b.Property("HasDeletedTimePeriods") + .HasColumnType("boolean"); + b.Property("LocationMappingPlan") .IsRequired() .HasColumnType("jsonb"); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs index e41e3027bc1..fa4d64d181b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs @@ -197,6 +197,191 @@ public async Task Success_MetaSummary() }, metaSummary.TimePeriodRange); } + + [Fact] + public async Task Success_HasDeletedIndicators_True() + { + var initialVersionMeta = new DataSetVersionMeta + { + IndicatorMetas = DataFixture.DefaultIndicatorMeta().GenerateList(2), + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta() + .WithLevels(ProcessorTestData.AbsenceSchool.ExpectedGeographicLevels) + }; + + var (instanceId, _, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage(), initialVersionMeta); + + await CreateMappings(instanceId); + + var mapping = await GetDataSetVersionMapping(nextVersion); + + Assert.True(mapping.HasDeletedIndicators); + } + + [Fact] + public async Task Success_HasDeletedIndicators_SameIndicators_False() + { + var initialVersionMeta = new DataSetVersionMeta + { + IndicatorMetas = ProcessorTestData.AbsenceSchool.ExpectedIndicators, + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta() + .WithLevels(ProcessorTestData.AbsenceSchool.ExpectedGeographicLevels) + }; + + var (instanceId, _, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage(), initialVersionMeta); + + await CreateMappings(instanceId); + + var mapping = await GetDataSetVersionMapping(nextVersion); + + Assert.False(mapping.HasDeletedIndicators); + } + + [Fact] + public async Task Success_HasDeletedIndicators_NewIndicators_False() + { + var initialVersionMeta = new DataSetVersionMeta + { + IndicatorMetas = ProcessorTestData.AbsenceSchool.ExpectedIndicators[..2], + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta() + .WithLevels(ProcessorTestData.AbsenceSchool.ExpectedGeographicLevels) + }; + + var (instanceId, _, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage(), initialVersionMeta); + + await CreateMappings(instanceId); + + var mapping = await GetDataSetVersionMapping(nextVersion); + + Assert.False(mapping.HasDeletedIndicators); + } + + [Fact] + public async Task Success_HasDeletedGeographicLevels_True() + { + var initialVersionMeta = new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta() + .WithLevels( + [ + GeographicLevel.Country, + GeographicLevel.Region, + GeographicLevel.LocalAuthority, + // Replaced by school in next version + GeographicLevel.Institution + ]) + }; + + var (instanceId, _, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage(), initialVersionMeta); + + await CreateMappings(instanceId); + + var mapping = await GetDataSetVersionMapping(nextVersion); + + Assert.True(mapping.HasDeletedGeographicLevels); + } + + [Fact] + public async Task Success_HasDeletedGeographicLevels_SameGeographicLevels_False() + { + var initialVersionMeta = new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta() + .WithLevels(ProcessorTestData.AbsenceSchool.ExpectedGeographicLevels) + }; + + var (instanceId, _, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage(), initialVersionMeta); + + await CreateMappings(instanceId); + + var mapping = await GetDataSetVersionMapping(nextVersion); + + Assert.False(mapping.HasDeletedGeographicLevels); + } + + [Fact] + public async Task Success_HasDeletedGeographicLevels_NewGeographicLevels_False() + { + var initialVersionMeta = new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta() + .WithLevels(ProcessorTestData.AbsenceSchool.ExpectedGeographicLevels[..2]) + }; + + var (instanceId, _, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage(), initialVersionMeta); + + await CreateMappings(instanceId); + + var mapping = await GetDataSetVersionMapping(nextVersion); + + Assert.False(mapping.HasDeletedGeographicLevels); + } + + [Fact] + public async Task Success_HasDeletedTimePeriods_True() + { + var initialVersionMeta = new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta() + .WithLevels(ProcessorTestData.AbsenceSchool.ExpectedGeographicLevels), + TimePeriodMetas = DataFixture.DefaultTimePeriodMeta() + .GenerateList(2) + }; + + var (instanceId, _, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage(), initialVersionMeta); + + await CreateMappings(instanceId); + + var mapping = await GetDataSetVersionMapping(nextVersion); + + Assert.True(mapping.HasDeletedTimePeriods); + } + + [Fact] + public async Task Success_HasDeletedTimePeriods_SameTimePeriods_False() + { + var initialVersionMeta = new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta() + .WithLevels(ProcessorTestData.AbsenceSchool.ExpectedGeographicLevels), + TimePeriodMetas = ProcessorTestData.AbsenceSchool.ExpectedTimePeriods + }; + + var (instanceId, _, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage(), initialVersionMeta); + + await CreateMappings(instanceId); + + var mapping = await GetDataSetVersionMapping(nextVersion); + + Assert.False(mapping.HasDeletedTimePeriods); + } + + [Fact] + public async Task Success_HasDeletedTimePeriods_NewTimePeriods_False() + { + var initialVersionMeta = new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta() + .WithLevels(ProcessorTestData.AbsenceSchool.ExpectedGeographicLevels), + TimePeriodMetas = ProcessorTestData.AbsenceSchool.ExpectedTimePeriods[..2] + }; + + var (instanceId, _, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage(), initialVersionMeta); + + await CreateMappings(instanceId); + + var mapping = await GetDataSetVersionMapping(nextVersion); + + Assert.False(mapping.HasDeletedTimePeriods); + } } public class CreateMappingsLocationsTests( @@ -1329,10 +1514,13 @@ private async Task CompleteProcessing(Guid instanceId) } private async Task<(Guid instanceId, DataSetVersion initialVersion, DataSetVersion nextVersion)> - CreateNextDataSetVersionAndDataFiles(DataSetVersionImportStage importStage) + CreateNextDataSetVersionAndDataFiles( + DataSetVersionImportStage importStage, + DataSetVersionMeta? initialVersionMeta = null) { var (initialDataSetVersion, nextDataSetVersion, instanceId) = await CreateDataSetInitialAndNextVersion( + initialVersionMeta: initialVersionMeta ?? GetDefaultInitialDataSetVersionMeta(), nextVersionImportStage: importStage, nextVersionStatus: DataSetVersionStatus.Processing); @@ -1365,6 +1553,15 @@ private async Task GetDataSetVersionMapping(DataSetVersio .SingleAsync(mapping => mapping.TargetDataSetVersionId == nextVersion.Id); } + private DataSetVersionMeta GetDefaultInitialDataSetVersionMeta() + { + return new DataSetVersionMeta + { + GeographicLevelMeta = DataFixture.DefaultGeographicLevelMeta() + .WithLevels(ProcessorTestData.AbsenceSchool.ExpectedGeographicLevels) + }; + } + private async Task AssertCorrectDataSetVersionNumbers(DataSetVersionMapping mapping, string expectedVersion) { Assert.Equal(expectedVersion, mapping.TargetDataSetVersion.SemVersion().ToString()); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/DataSertVersionMappingMeta.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/DataSertVersionMappingMeta.cs deleted file mode 100644 index a5fe594f124..00000000000 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Model/DataSertVersionMappingMeta.cs +++ /dev/null @@ -1,8 +0,0 @@ -using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; - -namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; - -public record DataSetVersionMappingMeta( - IDictionary> Filters, - IDictionary> Locations, - DataSetVersionMetaSummary MetaSummary); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Models/DataSetVersionMappingMeta.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Models/DataSetVersionMappingMeta.cs new file mode 100644 index 00000000000..047da672671 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Models/DataSetVersionMappingMeta.cs @@ -0,0 +1,18 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Models; + +public record DataSetVersionMappingMeta +{ + public required IDictionary> Filters { get; init; } + + public required IDictionary> Locations { get; init; } + + public required DataSetVersionMetaSummary MetaSummary { get; init; } + + public required IList Indicators { get; init; } + + public required GeographicLevelMeta GeographicLevel { get; init; } + + public required IList TimePeriods { get; init; } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/GeographicLevelMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/GeographicLevelMetaRepository.cs index 4dace1eb67d..57d10ce4de3 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/GeographicLevelMetaRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/GeographicLevelMetaRepository.cs @@ -50,11 +50,10 @@ FROM read_csv('{dataSetVersionPathResolver.CsvDataPath(dataSetVersion):raw}', AL .OrderBy(EnumToEnumLabelConverter.ToProvider) .ToList(); - var meta = new GeographicLevelMeta + return new GeographicLevelMeta { DataSetVersionId = dataSetVersion.Id, Levels = geographicLevels }; - return meta; } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorMetaRepository.cs index 40f19745822..ab6e28c29b0 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorMetaRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/IndicatorMetaRepository.cs @@ -14,6 +14,15 @@ public class IndicatorMetaRepository( PublicDataDbContext publicDataDbContext, IDataSetVersionPathResolver dataSetVersionPathResolver) : IIndicatorMetaRepository { + public async Task> ReadIndicatorMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + IReadOnlySet allowedColumns, + CancellationToken cancellationToken = default) + { + return await GetIndicatorMetas(duckDbConnection, dataSetVersion, allowedColumns, cancellationToken); + } + public async Task CreateIndicatorMetas( IDuckDbConnection duckDbConnection, DataSetVersion dataSetVersion, @@ -22,7 +31,7 @@ public async Task CreateIndicatorMetas( { var sourceDataSetVersionId = await GetSourceDataSetVersionId(dataSetVersion, cancellationToken); - var existingMetaIdsByColumn = sourceDataSetVersionId is not null + var existingPublicIdMappings = sourceDataSetVersionId is not null ? await publicDataDbContext.IndicatorMetas .Where(meta => meta.DataSetVersionId == sourceDataSetVersionId) .ToDictionaryAsync( @@ -32,10 +41,40 @@ public async Task CreateIndicatorMetas( ) : []; + var metas = await GetIndicatorMetas( + duckDbConnection, + dataSetVersion, + allowedColumns, + cancellationToken); + var currentId = await publicDataDbContext.NextSequenceValue( PublicDataDbContext.IndicatorMetasIdSequence, cancellationToken); + foreach (var meta in metas) + { + meta.Id = currentId++; + meta.PublicId = existingPublicIdMappings.TryGetValue(meta.Column, out var publicId) + ? publicId + : SqidEncoder.Encode(meta.Id); + } + + publicDataDbContext.IndicatorMetas.AddRange(metas); + await publicDataDbContext.SaveChangesAsync(cancellationToken); + + // Increase the sequence only by the amount that we used to generate new PublicIds. + await publicDataDbContext.SetSequenceValue( + PublicDataDbContext.IndicatorMetasIdSequence, + currentId - 1, + cancellationToken); + } + + private async Task> GetIndicatorMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + IReadOnlySet allowedColumns, + CancellationToken cancellationToken) + { var metaRows = await duckDbConnection.SqlBuilder( $""" SELECT * @@ -45,33 +84,18 @@ public async Task CreateIndicatorMetas( """) .QueryAsync(cancellationToken: cancellationToken); - var metas = metaRows + return metaRows .OrderBy(row => row.Label) - .Select(row => + .Select(row => new IndicatorMeta { - var id = currentId++; - - return new IndicatorMeta - { - Id = id, - DataSetVersionId = dataSetVersion.Id, - PublicId = existingMetaIdsByColumn.GetValueOrDefault(row.ColName, SqidEncoder.Encode(id)), - Column = row.ColName, - Label = row.Label, - Unit = row.ParsedIndicatorUnit, - DecimalPlaces = row.IndicatorDp - }; + DataSetVersionId = dataSetVersion.Id, + PublicId = string.Empty, + Column = row.ColName, + Label = row.Label, + Unit = row.ParsedIndicatorUnit, + DecimalPlaces = row.IndicatorDp }) .ToList(); - - publicDataDbContext.IndicatorMetas.AddRange(metas); - await publicDataDbContext.SaveChangesAsync(cancellationToken); - - // Increase the sequence only by the amount that we used to generate new PublicIds. - await publicDataDbContext.SetSequenceValue( - PublicDataDbContext.IndicatorMetasIdSequence, - currentId - 1, - cancellationToken); } private async Task GetSourceDataSetVersionId( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IIndicatorMetaRepository.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IIndicatorMetaRepository.cs index c0d6643fb75..39be08003d9 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IIndicatorMetaRepository.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Repository/Interfaces/IIndicatorMetaRepository.cs @@ -5,6 +5,12 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repos public interface IIndicatorMetaRepository { + Task> ReadIndicatorMetas( + IDuckDbConnection duckDbConnection, + DataSetVersion dataSetVersion, + IReadOnlySet allowedColumns, + CancellationToken cancellationToken = default); + Task CreateIndicatorMetas( IDuckDbConnection duckDbConnection, DataSetVersion dataSetVersion, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetMetaService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetMetaService.cs index a205af7ae0b..aab05c6b37a 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetMetaService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetMetaService.cs @@ -3,7 +3,6 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.DuckDb; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Models; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Repository.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; @@ -27,7 +26,7 @@ public class DataSetMetaService( ITimePeriodsDuckDbRepository timePeriodsDuckDbRepository ) : IDataSetMetaService { - public async Task ReadDataSetVersionMetaForMappings( + public async Task ReadDataSetVersionMappingMeta( Guid dataSetVersionId, CancellationToken cancellationToken = default) { @@ -58,6 +57,12 @@ public async Task ReadDataSetVersionMetaForMappings( dataSetVersion, cancellationToken); + var indicatorMetas = await indicatorMetaRepository.ReadIndicatorMetas( + duckDbConnection, + dataSetVersion, + allowedColumns, + cancellationToken); + var locationMetas = await locationMetaRepository.ReadLocationMetas( duckDbConnection, dataSetVersion, @@ -75,7 +80,15 @@ public async Task ReadDataSetVersionMetaForMappings( allowedColumns, geographicLevelMeta); - return new DataSetVersionMappingMeta(filterMetas, locationMetas, metaSummary); + return new DataSetVersionMappingMeta + { + Filters = filterMetas, + Locations = locationMetas, + MetaSummary = metaSummary, + Indicators = indicatorMetas, + GeographicLevel = geographicLevelMeta, + TimePeriods = timePeriodMetas + }; } public async Task CreateDataSetVersionMeta( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionMappingService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionMappingService.cs index 0dd6f2fe5e8..252b8b2b4a4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionMappingService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionMappingService.cs @@ -45,7 +45,7 @@ public async Task> CreateMappings( var liveVersion = nextVersion.DataSet.LatestLiveVersion!; - var nextVersionMeta = await dataSetMetaService.ReadDataSetVersionMetaForMappings( + var nextVersionMeta = await dataSetMetaService.ReadDataSetVersionMappingMeta( dataSetVersionId: nextDataSetVersionId, cancellationToken); @@ -63,6 +63,21 @@ public async Task> CreateMappings( sourceFilterMeta, nextVersionMeta.Filters); + var hasDeletedIndicators = await HasDeletedIndicators( + liveVersion.Id, + nextVersionMeta.Indicators, + cancellationToken); + + var hasDeletedGeographicLevels = await HasDeletedGeographicLevels( + liveVersion.Id, + nextVersionMeta.GeographicLevel, + cancellationToken); + + var hasDeletedTimePeriods = await HasDeletedTimePeriods( + liveVersion.Id, + nextVersionMeta.TimePeriods, + cancellationToken); + nextVersion.MetaSummary = nextVersionMeta.MetaSummary; publicDataDbContext @@ -72,7 +87,10 @@ public async Task> CreateMappings( SourceDataSetVersionId = liveVersion.Id, TargetDataSetVersionId = nextDataSetVersionId, LocationMappingPlan = locationMappings, - FilterMappingPlan = filterMappings + FilterMappingPlan = filterMappings, + HasDeletedIndicators = hasDeletedIndicators, + HasDeletedGeographicLevels = hasDeletedGeographicLevels, + HasDeletedTimePeriods = hasDeletedTimePeriods }); await publicDataDbContext.SaveChangesAsync(cancellationToken); @@ -466,8 +484,7 @@ private async Task> GetLocationMeta( Guid dataSetVersionId, CancellationToken cancellationToken) { - return await publicDataDbContext - .LocationMetas + return await publicDataDbContext.LocationMetas .AsNoTracking() .Include(meta => meta.OptionLinks) .ThenInclude(link => link.Option) @@ -477,8 +494,7 @@ private async Task> GetLocationMeta( private async Task> GetFilterMeta(Guid dataSetVersionId, CancellationToken cancellationToken) { - return await publicDataDbContext - .FilterMetas + return await publicDataDbContext.FilterMetas .AsNoTracking() .Include(meta => meta.OptionLinks) .ThenInclude(link => link.Option) @@ -486,6 +502,48 @@ private async Task> GetFilterMeta(Guid dataSetVersionId, Cancel .ToListAsync(cancellationToken); } + private async Task HasDeletedIndicators( + Guid dataSetVersionId, + IEnumerable newIndicatorMetas, + CancellationToken cancellationToken) + { + return (await publicDataDbContext.IndicatorMetas + .AsNoTracking() + .Where(meta => meta.DataSetVersionId == dataSetVersionId) + .Select(meta => meta.Column) + .ToListAsync(cancellationToken)) + .Except(newIndicatorMetas.Select(meta => meta.Column)) + .Any(); + } + + private async Task HasDeletedGeographicLevels( + Guid dataSetVersionId, + GeographicLevelMeta newGeographicLevelMeta, + CancellationToken cancellationToken) + { + var oldGeographicLevelMeta = await publicDataDbContext.GeographicLevelMetas + .AsNoTracking() + .SingleAsync(meta => meta.DataSetVersionId == dataSetVersionId, cancellationToken); + + return oldGeographicLevelMeta.Levels + .Except(newGeographicLevelMeta.Levels) + .Any(); + } + + private async Task HasDeletedTimePeriods( + Guid dataSetVersionId, + IEnumerable newTimePeriodMetas, + CancellationToken cancellationToken) + { + return (await publicDataDbContext.TimePeriodMetas + .AsNoTracking() + .Where(meta => meta.DataSetVersionId == dataSetVersionId) + .Select(m => $"{m.Period}|{m.Code}") + .ToListAsync(cancellationToken)) + .Except(newTimePeriodMetas.Select(m => $"{m.Period}|{m.Code}")) + .Any(); + } + private static BadRequestObjectResult CreateDataSetVersionIdError( LocalizableMessage message, Guid dataSetVersionId, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetMetaService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetMetaService.cs index c6005a98904..6dae78f6906 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetMetaService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/Interfaces/IDataSetMetaService.cs @@ -1,10 +1,10 @@ -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Models; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; public interface IDataSetMetaService { - Task ReadDataSetVersionMetaForMappings( + Task ReadDataSetVersionMappingMeta( Guid dataSetVersionId, CancellationToken cancellationToken = default); From 2f5dcbc2fc7e7b22f8223dd840985366e92b158d Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Sat, 12 Oct 2024 01:43:01 +0100 Subject: [PATCH 63/80] EES-5563 Add handling of meta deletion flags in mapping update endpoints --- .../DataSetVersionMappingControllerTests.cs | 289 +++++++++++++++++- .../DataSetVersionMappingService.cs | 182 +++++------ 2 files changed, 371 insertions(+), 100 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetVersionMappingControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetVersionMappingControllerTests.cs index bfdcde74470..68bd16c5e34 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetVersionMappingControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin.Tests/Controllers/Api/Public.Data/DataSetVersionMappingControllerTests.cs @@ -416,7 +416,7 @@ public async Task Success_MappingsCompleteAndVersionUpdated( } [Fact] - public async Task Success_DeletedLevel_MajorUpdate() + public async Task Success_MappedLocation_HasDeletedLocationLevels_MajorUpdate() { var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); @@ -482,7 +482,7 @@ public async Task Success_DeletedLevel_MajorUpdate() } [Fact] - public async Task Success_AddedLevel_MinorUpdate() + public async Task Success_MappedLocation_HasDeletedIndicators_MajorUpdate() { var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); @@ -504,16 +504,125 @@ public async Task Success_AddedLevel_MinorUpdate() .WithNoMapping()) .AddCandidate( targetKey: "target-location-1-key", - candidate: DataFixture.DefaultMappableLocationOption())) - // Country level has been added and only has candidates. + candidate: DataFixture.DefaultMappableLocationOption()))) + // There are deleted indicators that cannot be mapped. + .WithHasDeletedIndicators(true); + + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + List updates = + [ + new() + { + Level = GeographicLevel.LocalAuthority, + SourceKey = "source-location-1-key", + Type = MappingType.ManualMapped, + CandidateKey = "target-location-1-key" + } + ]; + + var response = await ApplyBatchLocationMappingUpdates( + nextDataSetVersionId: nextDataSetVersion.Id, + updates: updates); + + response.AssertOk(); + + var updatedMapping = await TestApp.GetDbContext() + .DataSetVersionMappings + .Include(m => m.TargetDataSetVersion) + .SingleAsync(m => m.TargetDataSetVersionId == nextDataSetVersion.Id); + + Assert.True(updatedMapping.LocationMappingsComplete); + + // This update completes the mapping and would normally be a minor version + // update, but the deleted indicators mean this is still a major version update. + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); + } + + [Fact] + public async Task Success_MappedLocation_HasDeletedGeographicLevels_MajorUpdate() + { + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); + + DataSetVersionMapping mapping = DataFixture + .DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(initialDataSetVersion.Id) + .WithTargetDataSetVersionId(nextDataSetVersion.Id) + .WithLocationMappingPlan(DataFixture + .DefaultLocationMappingPlan() .AddLevel( - level: GeographicLevel.Country, + level: GeographicLevel.LocalAuthority, + mappings: DataFixture + .DefaultLocationLevelMappings() + .AddMapping( + sourceKey: "source-location-1-key", + mapping: DataFixture + .DefaultLocationOptionMapping() + .WithSource(DataFixture.DefaultMappableLocationOption()) + .WithNoMapping()) + .AddCandidate( + targetKey: "target-location-1-key", + candidate: DataFixture.DefaultMappableLocationOption()))) + // There are deleted geographic levels that cannot be mapped. + .WithHasDeletedGeographicLevels(true); + + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + List updates = + [ + new() + { + Level = GeographicLevel.LocalAuthority, + SourceKey = "source-location-1-key", + Type = MappingType.ManualMapped, + CandidateKey = "target-location-1-key" + } + ]; + + var response = await ApplyBatchLocationMappingUpdates( + nextDataSetVersionId: nextDataSetVersion.Id, + updates: updates); + + response.AssertOk(); + + var updatedMapping = await TestApp.GetDbContext() + .DataSetVersionMappings + .Include(m => m.TargetDataSetVersion) + .SingleAsync(m => m.TargetDataSetVersionId == nextDataSetVersion.Id); + + Assert.True(updatedMapping.LocationMappingsComplete); + + // This update completes the mapping and would normally be a minor version + // update, but the deleted geographic levels mean this is still a major version update. + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); + } + + [Fact] + public async Task Success_MappedLocation_HasDeletedTimePeriods_MajorUpdate() + { + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); + + DataSetVersionMapping mapping = DataFixture + .DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(initialDataSetVersion.Id) + .WithTargetDataSetVersionId(nextDataSetVersion.Id) + .WithLocationMappingPlan(DataFixture + .DefaultLocationMappingPlan() + .AddLevel( + level: GeographicLevel.LocalAuthority, mappings: DataFixture .DefaultLocationLevelMappings() + .AddMapping( + sourceKey: "source-location-1-key", + mapping: DataFixture + .DefaultLocationOptionMapping() + .WithSource(DataFixture.DefaultMappableLocationOption()) + .WithNoMapping()) .AddCandidate( - targetKey: "source-location-1-key", - candidate: DataFixture - .DefaultMappableLocationOption()))); + targetKey: "target-location-1-key", + candidate: DataFixture.DefaultMappableLocationOption()))) + // There are deleted time periods that cannot be mapped. + .WithHasDeletedTimePeriods(true); await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); @@ -539,13 +648,14 @@ public async Task Success_AddedLevel_MinorUpdate() .Include(m => m.TargetDataSetVersion) .SingleAsync(m => m.TargetDataSetVersionId == nextDataSetVersion.Id); - // This update completes the mapping as a location level was added - // and isn't considered as needing to be mapped - minor version update. Assert.True(updatedMapping.LocationMappingsComplete); - await AssertCorrectDataSetVersionNumbers(updatedMapping, "1.1.0"); + // This update completes the mapping and would normally be a minor version + // update, but the deleted time periods mean this is still a major version update. + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); } + [Fact] public async Task SourceKeyDoesNotExist_Returns400_AndRollsBackTransaction() { @@ -1304,6 +1414,163 @@ public async Task Success_VersionUpdate( await AssertCorrectDataSetVersionNumbers(updatedMapping, expectedVersion); } + [Fact] + public async Task Success_VersionUpdate_HasDeletedIndicators_MajorUpdate() + { + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); + + DataSetVersionMapping mapping = DataFixture + .DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(initialDataSetVersion.Id) + .WithTargetDataSetVersionId(nextDataSetVersion.Id) + .WithFilterMappingPlan(DataFixture + .DefaultFilterMappingPlan() + .AddFilterMapping("filter-1-key", DataFixture + .DefaultFilterMapping() + .WithAutoMapped("filter-1-key") + .AddOptionMapping("filter-1-option-1-key", DataFixture + .DefaultFilterOptionMapping() + .WithNoMapping())) + .AddFilterCandidate("filter-1-key", DataFixture + .DefaultFilterMappingCandidate() + .AddOptionCandidate("filter-1-option-1-key", DataFixture + .DefaultMappableFilterOption()))) + // Has deleted indicators that cannot be mapped + .WithHasDeletedIndicators(true); + + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + List updates = + [ + new() + { + FilterKey = "filter-1-key", + SourceKey = "filter-1-option-1-key", + Type = MappingType.ManualMapped, + CandidateKey = "filter-1-option-1-key" + } + ]; + + var response = await ApplyBatchFilterOptionMappingUpdates( + nextDataSetVersionId: nextDataSetVersion.Id, + updates: updates); + + response.AssertOk(); + + var updatedMapping = await TestApp.GetDbContext() + .DataSetVersionMappings + .Include(m => m.TargetDataSetVersion) + .SingleAsync(m => m.TargetDataSetVersionId == nextDataSetVersion.Id); + + // Should be a minor version update, but has deleted indicators so must be a major update. + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); + } + + [Fact] + public async Task Success_VersionUpdate_HasDeletedGeographicLevels_MajorUpdate() + { + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); + + DataSetVersionMapping mapping = DataFixture + .DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(initialDataSetVersion.Id) + .WithTargetDataSetVersionId(nextDataSetVersion.Id) + .WithFilterMappingPlan(DataFixture + .DefaultFilterMappingPlan() + .AddFilterMapping("filter-1-key", DataFixture + .DefaultFilterMapping() + .WithAutoMapped("filter-1-key") + .AddOptionMapping("filter-1-option-1-key", DataFixture + .DefaultFilterOptionMapping() + .WithNoMapping())) + .AddFilterCandidate("filter-1-key", DataFixture + .DefaultFilterMappingCandidate() + .AddOptionCandidate("filter-1-option-1-key", DataFixture + .DefaultMappableFilterOption()))) + // Has deleted geographic levels that cannot be mapped + .WithHasDeletedGeographicLevels(true); + + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + List updates = + [ + new() + { + FilterKey = "filter-1-key", + SourceKey = "filter-1-option-1-key", + Type = MappingType.ManualMapped, + CandidateKey = "filter-1-option-1-key" + } + ]; + + var response = await ApplyBatchFilterOptionMappingUpdates( + nextDataSetVersionId: nextDataSetVersion.Id, + updates: updates); + + response.AssertOk(); + + var updatedMapping = await TestApp.GetDbContext() + .DataSetVersionMappings + .Include(m => m.TargetDataSetVersion) + .SingleAsync(m => m.TargetDataSetVersionId == nextDataSetVersion.Id); + + // Should be a minor version update, but has deleted geographic levels so must be a major update. + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); + } + + [Fact] + public async Task Success_VersionUpdate_HasDeletedTimePeriods_MajorUpdate() + { + var (initialDataSetVersion, nextDataSetVersion) = await CreateInitialAndNextDataSetVersion(); + + DataSetVersionMapping mapping = DataFixture + .DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(initialDataSetVersion.Id) + .WithTargetDataSetVersionId(nextDataSetVersion.Id) + .WithFilterMappingPlan(DataFixture + .DefaultFilterMappingPlan() + .AddFilterMapping("filter-1-key", DataFixture + .DefaultFilterMapping() + .WithAutoMapped("filter-1-key") + .AddOptionMapping("filter-1-option-1-key", DataFixture + .DefaultFilterOptionMapping() + .WithNoMapping())) + .AddFilterCandidate("filter-1-key", DataFixture + .DefaultFilterMappingCandidate() + .AddOptionCandidate( + "filter-1-option-1-key", + DataFixture.DefaultMappableFilterOption()))) + // Has deleted time periods that cannot be mapped + .WithHasDeletedTimePeriods(true); + + await TestApp.AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + List updates = + [ + new() + { + FilterKey = "filter-1-key", + SourceKey = "filter-1-option-1-key", + Type = MappingType.ManualMapped, + CandidateKey = "filter-1-option-1-key" + } + ]; + + var response = await ApplyBatchFilterOptionMappingUpdates( + nextDataSetVersionId: nextDataSetVersion.Id, + updates: updates); + + response.AssertOk(); + + var updatedMapping = await TestApp.GetDbContext() + .DataSetVersionMappings + .Include(m => m.TargetDataSetVersion) + .SingleAsync(m => m.TargetDataSetVersionId == nextDataSetVersion.Id); + + // Should be a minor version update, but has deleted time periods so must be a major update. + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); + } + [Theory] [InlineData(MappingType.ManualMapped)] [InlineData(MappingType.ManualNone)] diff --git a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/DataSetVersionMappingService.cs b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/DataSetVersionMappingService.cs index 9c582c5760d..de64f5c7551 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/DataSetVersionMappingService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Admin/Services/Public.Data/DataSetVersionMappingService.cs @@ -134,43 +134,33 @@ await userService private async Task UpdateMappingsCompleteAndVersion(Guid nextDataSetVersionId, CancellationToken cancellationToken) { - var hasDeletedLocationLevels = await HasDeletedLocationLevels( - nextDataSetVersionId: nextDataSetVersionId, - cancellationToken: cancellationToken); - - var locationMappingTypes = await GetLocationOptionMappingTypes( - nextDataSetVersionId: nextDataSetVersionId, + var locationOptionMappingTypes = await GetLocationOptionMappingTypes( + targetDataSetVersionId: nextDataSetVersionId, cancellationToken: cancellationToken); var filterAndOptionMappingTypes = await GetFilterAndOptionMappingTypes( - nextDataSetVersionId: nextDataSetVersionId, + targetDataSetVersionId: nextDataSetVersionId, cancellationToken: cancellationToken); await UpdateMappingCompleteFlags( nextDataSetVersionId: nextDataSetVersionId, - locationMappingTypes: locationMappingTypes, + locationLevelAndOptionMappingTypes: locationOptionMappingTypes, filterAndOptionMappingTypes: filterAndOptionMappingTypes, cancellationToken: cancellationToken); await UpdateVersionNumber( nextDataSetVersionId: nextDataSetVersionId, - hasDeletedLocationLevels: hasDeletedLocationLevels, - locationMappingTypes: locationMappingTypes, - filterAndOptionMappingTypes: filterAndOptionMappingTypes, + locationMappingTypes: locationOptionMappingTypes, + filterMappingTypes: filterAndOptionMappingTypes, cancellationToken: cancellationToken); } private async Task UpdateVersionNumber( Guid nextDataSetVersionId, - bool hasDeletedLocationLevels, - List locationMappingTypes, - List filterAndOptionMappingTypes, + List locationMappingTypes, + List filterMappingTypes, CancellationToken cancellationToken) { - var hasUnmappedOptions = locationMappingTypes - .Concat(filterAndOptionMappingTypes.Select(mappings => mappings.OptionMappingType)) - .Any(type => NoMappingTypes.Contains(type)); - var sourceDataSetVersion = await publicDataDbContext .DataSetVersionMappings .Where(mapping => mapping.TargetDataSetVersionId == nextDataSetVersionId) @@ -183,7 +173,11 @@ private async Task UpdateVersionNumber( .Select(nextVersion => nextVersion.TargetDataSetVersion) .SingleAsync(cancellationToken); - var isMajorVersionUpdate = hasDeletedLocationLevels || hasUnmappedOptions; + var isMajorVersionUpdate = await IsMajorVersionUpdate( + nextDataSetVersionId, + locationMappingTypes, + filterMappingTypes, + cancellationToken); if (isMajorVersionUpdate) { @@ -208,27 +202,53 @@ private async Task UpdateVersionNumber( await contentDbContext.SaveChangesAsync(cancellationToken); } + private async Task IsMajorVersionUpdate( + Guid targetDataSetVersionId, + List locationMappingTypes, + List filterMappingTypes, + CancellationToken cancellationToken) + { + if (locationMappingTypes.Any(types => NoMappingTypes.Contains(types.LocationLevel) + || NoMappingTypes.Contains(types.LocationOption))) + { + return true; + } + + if (filterMappingTypes.Any(types => NoMappingTypes.Contains(types.Filter) + || NoMappingTypes.Contains(types.FilterOption))) + { + return true; + } + + return await publicDataDbContext.DataSetVersionMappings + .Where(mapping => mapping.TargetDataSetVersionId == targetDataSetVersionId) + .Select(mapping => mapping.HasDeletedIndicators + || mapping.HasDeletedGeographicLevels + || mapping.HasDeletedTimePeriods) + .SingleOrDefaultAsync(cancellationToken); + } + private async Task UpdateMappingCompleteFlags( Guid nextDataSetVersionId, - List locationMappingTypes, - List filterAndOptionMappingTypes, + List locationLevelAndOptionMappingTypes, + List filterAndOptionMappingTypes, CancellationToken cancellationToken) { // Find any location options that have a mapping type that indicates the user // still needs to take action in order to resolve the mapping. - var locationMappingsComplete = !locationMappingTypes - .Any(type => IncompleteMappingTypes.Contains(type)); + // We omit options for location levels that are mapped as `AutoNone` as these + // means the entire location level has been deleted and cannot be mapped. + var locationMappingsComplete = !locationLevelAndOptionMappingTypes + .Where(types => types.LocationLevel != MappingType.AutoNone) + .Any(types => IncompleteMappingTypes.Contains(types.LocationOption)); // Find any filter options that that indicates the user still needs to take action // in order to resolve the mapping. If any exist, mappings are not yet complete. - // - // We do however omit checking the filter options of filters that have a mapping of - // "AutoNone", as currently there is no way within the UI for the users to handle - // the resolution of these unmapped filters, and so without ignoring these, the user - // would never be able to complete the mappings. + // We omit options for filters that are mapped as `AutoNone` as this + // means the entire filter has been deleted and cannot be mapped. var filterMappingsComplete = !filterAndOptionMappingTypes - .Where(types => types.FilterMappingType != MappingType.AutoNone) - .Any(types => IncompleteMappingTypes.Contains(types.OptionMappingType)); + .Where(types => types.Filter != MappingType.AutoNone) + .Any(types => IncompleteMappingTypes.Contains(types.FilterOption)); // Update the mapping complete flags. await publicDataDbContext @@ -241,40 +261,23 @@ await publicDataDbContext cancellationToken: cancellationToken); } - private async Task HasDeletedLocationLevels( - Guid nextDataSetVersionId, - CancellationToken cancellationToken) - { - var targetDataSetVersionIdParam = new NpgsqlParameter("targetDataSetVersionId", nextDataSetVersionId); - - var deletedLevelCount = await publicDataDbContext.Database - .SqlQueryRaw( - $$$""" - SELECT DISTINCT COUNT(Level.key) "Value" - FROM "{{{nameof(PublicDataDbContext.DataSetVersionMappings)}}}" Mapping, - jsonb_each(Mapping."{{{nameof(DataSetVersionMapping.LocationMappingPlan)}}}" - -> '{{{nameof(LocationMappingPlan.Levels)}}}') Level - WHERE "{{{nameof(DataSetVersionMapping.TargetDataSetVersionId)}}}" = @targetDataSetVersionId - AND Level.value -> '{{{nameof(LocationLevelMappings.Candidates)}}}' = '{{}}'::jsonb - """, - parameters: [targetDataSetVersionIdParam]) - .FirstAsync(cancellationToken); - - return deletedLevelCount > 0; - } - - private async Task> GetLocationOptionMappingTypes( - Guid nextDataSetVersionId, + private async Task> GetLocationOptionMappingTypes( + Guid targetDataSetVersionId, CancellationToken cancellationToken) { - var targetDataSetVersionIdParam = new NpgsqlParameter("targetDataSetVersionId", nextDataSetVersionId); + var targetDataSetVersionIdParam = new NpgsqlParameter("targetDataSetVersionId", targetDataSetVersionId); // Find the distinct mapping types for location options across location levels // that still have candidates (and haven't been deleted). - var types = await publicDataDbContext.Database - .SqlQueryRaw( + return await publicDataDbContext.Database + .SqlQueryRaw( $$$""" - SELECT DISTINCT OptionMappingType "Value" + SELECT DISTINCT + CASE WHEN Level.value -> '{{{nameof(LocationLevelMappings.Candidates)}}}' = '{{}}' + THEN '{{{nameof(MappingType.AutoNone)}}}' + ELSE '{{{nameof(MappingType.AutoMapped)}}}' + END "{{{nameof(LocationMappingTypes.LocationLevelRaw)}}}", + OptionMappingType "{{{nameof(LocationMappingTypes.LocationOptionRaw)}}}" FROM "{{{nameof(PublicDataDbContext.DataSetVersionMappings)}}}" Mapping, jsonb_each(Mapping."{{{nameof(DataSetVersionMapping.LocationMappingPlan)}}}" @@ -282,44 +285,34 @@ SELECT DISTINCT OptionMappingType "Value" jsonb_each(Level.value -> '{{{nameof(LocationLevelMappings.Mappings)}}}') OptionMapping, jsonb_extract_path_text(OptionMapping.value, '{{{nameof(LocationOptionMapping.Type)}}}') OptionMappingType WHERE "{{{nameof(DataSetVersionMapping.TargetDataSetVersionId)}}}" = @targetDataSetVersionId - AND Level.value -> '{{{nameof(LocationLevelMappings.Candidates)}}}' != '{{}}'::jsonb - AND Level.value -> '{{{nameof(LocationLevelMappings.Mappings)}}}' != '{{}}'::jsonb """, parameters: [targetDataSetVersionIdParam]) .ToListAsync(cancellationToken); - - return types - .Select(EnumUtil.GetFromEnumValue) - .ToList(); } - private async Task> GetFilterAndOptionMappingTypes( - Guid nextDataSetVersionId, + private async Task> GetFilterAndOptionMappingTypes( + Guid targetDataSetVersionId, CancellationToken cancellationToken) { - var targetDataSetVersionIdParam = new NpgsqlParameter("targetDataSetVersionId", nextDataSetVersionId); + var targetDataSetVersionIdParam = new NpgsqlParameter("targetDataSetVersionId", targetDataSetVersionId); // Find all the distinct combinations of parent filters' mapping types against the distinct // mapping types of their children. return await publicDataDbContext.Database - .SqlQueryRaw( + .SqlQueryRaw( $""" - SELECT DISTINCT - FilterMappingType "{nameof(FilterAndOptionMappingTypes.FilterMappingTypeRaw)}", - OptionMappingType "{nameof(FilterAndOptionMappingTypes.OptionMappingTypeRaw)}" - FROM ( - SELECT FilterMappingType, - OptionMappingType - FROM - "{nameof(PublicDataDbContext.DataSetVersionMappings)}" Mapping, - jsonb_each(Mapping."{nameof(DataSetVersionMapping.FilterMappingPlan)}" - -> '{nameof(FilterMappingPlan.Mappings)}') FilterMapping, - jsonb_each(FilterMapping.value -> '{nameof(FilterMapping.OptionMappings)}') OptionMapping, - jsonb_extract_path_text(FilterMapping.value, '{nameof(FilterMapping.Type)}') FilterMappingType, - jsonb_extract_path_text(OptionMapping.value, '{nameof(FilterOptionMapping.Type)}') OptionMappingType - WHERE "{nameof(DataSetVersionMapping.TargetDataSetVersionId)}" = @targetDataSetVersionId - ) - """, + SELECT DISTINCT + FilterMappingType "{nameof(FilterMappingTypes.FilterRaw)}", + OptionMappingType "{nameof(FilterMappingTypes.FilterOptionRaw)}" + FROM + "{nameof(PublicDataDbContext.DataSetVersionMappings)}" Mapping, + jsonb_each(Mapping."{nameof(DataSetVersionMapping.FilterMappingPlan)}" + -> '{nameof(FilterMappingPlan.Mappings)}') FilterMapping, + jsonb_each(FilterMapping.value -> '{nameof(FilterMapping.OptionMappings)}') OptionMapping, + jsonb_extract_path_text(FilterMapping.value, '{nameof(FilterMapping.Type)}') FilterMappingType, + jsonb_extract_path_text(OptionMapping.value, '{nameof(FilterOptionMapping.Type)}') OptionMappingType + WHERE "{nameof(DataSetVersionMapping.TargetDataSetVersionId)}" = @targetDataSetVersionId + """, parameters: [targetDataSetVersionIdParam]) .ToListAsync(cancellationToken); } @@ -621,14 +614,25 @@ private async Task> CheckMappingExis cancellationToken); } - private record FilterAndOptionMappingTypes + private record FilterMappingTypes + { + public required string FilterRaw { get; init; } + + public required string FilterOptionRaw { get; init; } + + public MappingType Filter => EnumUtil.GetFromEnumValue(FilterRaw); + + public MappingType FilterOption => EnumUtil.GetFromEnumValue(FilterOptionRaw); + } + + private record LocationMappingTypes { - public required string FilterMappingTypeRaw { get; init; } + public required string LocationLevelRaw { get; init; } - public required string OptionMappingTypeRaw { get; init; } + public required string LocationOptionRaw { get; init; } - public MappingType FilterMappingType => EnumUtil.GetFromEnumValue(FilterMappingTypeRaw); + public MappingType LocationLevel => EnumUtil.GetFromEnumValue(LocationLevelRaw); - public MappingType OptionMappingType => EnumUtil.GetFromEnumValue(OptionMappingTypeRaw); + public MappingType LocationOption => EnumUtil.GetFromEnumValue(LocationOptionRaw); } } From a26a348b86fb36e7900273d0f1b773408d0b4d6b Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Sun, 13 Oct 2024 18:03:29 +0100 Subject: [PATCH 64/80] EES-5563 Add handling of meta deletion flags in `ApplyAutoMappings` function --- ...NextDataSetVersionMappingsFunctionTests.cs | 290 +++++++++++++++++- .../Services/DataSetVersionMappingService.cs | 69 +++-- 2 files changed, 331 insertions(+), 28 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs index fa4d64d181b..77d1b1f78b4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs @@ -663,6 +663,69 @@ await AddTestData(context => Assert.Equal(Stage, savedImport.Stage); Assert.Equal(DataSetVersionStatus.Processing, savedImport.DataSetVersion.Status); } + + [Fact] + public async Task Success_HasDeletedIndicators_MajorUpdate() + { + var (instanceId, originalVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + DataSetVersionMapping mapping = DataFixture + .DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(originalVersion.Id) + .WithTargetDataSetVersionId(nextVersion.Id) + .WithHasDeletedIndicators(true); + + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ApplyAutoMappings(instanceId); + + var updatedMapping = await GetDataSetVersionMapping(nextVersion); + + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); + } + + [Fact] + public async Task Success_HasDeletedGeographicLevels_MajorUpdate() + { + var (instanceId, originalVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + DataSetVersionMapping mapping = DataFixture + .DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(originalVersion.Id) + .WithTargetDataSetVersionId(nextVersion.Id) + .WithHasDeletedGeographicLevels(true); + + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ApplyAutoMappings(instanceId); + + var updatedMapping = await GetDataSetVersionMapping(nextVersion); + + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); + } + + [Fact] + public async Task Success_HasDeletedTimePeriods_MajorUpdate() + { + var (instanceId, originalVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + DataSetVersionMapping mapping = DataFixture + .DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(originalVersion.Id) + .WithTargetDataSetVersionId(nextVersion.Id) + .WithHasDeletedTimePeriods(true); + + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ApplyAutoMappings(instanceId); + + var updatedMapping = await GetDataSetVersionMapping(nextVersion); + + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); + } } public class ApplyAutoMappingsLocationsTests( @@ -1087,6 +1150,120 @@ public async Task Complete_AutoMappedAndNewOptions_MinorUpdate() // All source options have auto-mapped candidates - minor version update. await AssertCorrectDataSetVersionNumbers(updatedMapping, "1.1.0"); } + + [Fact] + public async Task Complete_HasDeletedIndicators_MajorUpdate() + { + var (instanceId, originalVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + DataSetVersionMapping mapping = DataFixture + .DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(originalVersion.Id) + .WithTargetDataSetVersionId(nextVersion.Id) + .WithLocationMappingPlan(DataFixture + .DefaultLocationMappingPlan() + .AddLevel( + level: GeographicLevel.LocalAuthority, + mappings: DataFixture + .DefaultLocationLevelMappings() + .AddMapping( + sourceKey: "location-1-key", + mapping: DataFixture + .DefaultLocationOptionMapping() + .WithSource(DataFixture.DefaultMappableLocationOption()) + .WithNoMapping()) + .AddCandidate( + targetKey: "location-1-key", + candidate: DataFixture.DefaultMappableLocationOption()))) + // Has deleted indicators that cannot be mapped + .WithHasDeletedIndicators(true); + + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ApplyAutoMappings(instanceId); + + var updatedMapping = await GetDataSetVersionMapping(nextVersion); + + // Is an exact match but as there are deleted indicators so needs to be a major update. + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); + } + + [Fact] + public async Task Complete_HasDeletedGeographicLevels_MajorUpdate() + { + var (instanceId, originalVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + DataSetVersionMapping mapping = DataFixture + .DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(originalVersion.Id) + .WithTargetDataSetVersionId(nextVersion.Id) + .WithLocationMappingPlan(DataFixture + .DefaultLocationMappingPlan() + .AddLevel( + level: GeographicLevel.LocalAuthority, + mappings: DataFixture + .DefaultLocationLevelMappings() + .AddMapping( + sourceKey: "location-1-key", + mapping: DataFixture + .DefaultLocationOptionMapping() + .WithSource(DataFixture.DefaultMappableLocationOption()) + .WithNoMapping()) + .AddCandidate( + targetKey: "location-1-key", + candidate: DataFixture.DefaultMappableLocationOption()))) + // Has deleted geographic levels that cannot be mapped + .WithHasDeletedGeographicLevels(true); + + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ApplyAutoMappings(instanceId); + + var updatedMapping = await GetDataSetVersionMapping(nextVersion); + + // Is an exact match but as there are deleted geographic levels so needs to be a major update. + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); + } + + [Fact] + public async Task Complete_HasDeletedTimePeriods_MajorUpdate() + { + var (instanceId, originalVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + DataSetVersionMapping mapping = DataFixture + .DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(originalVersion.Id) + .WithTargetDataSetVersionId(nextVersion.Id) + .WithLocationMappingPlan(DataFixture + .DefaultLocationMappingPlan() + .AddLevel( + level: GeographicLevel.LocalAuthority, + mappings: DataFixture + .DefaultLocationLevelMappings() + .AddMapping( + sourceKey: "location-1-key", + mapping: DataFixture + .DefaultLocationOptionMapping() + .WithSource(DataFixture.DefaultMappableLocationOption()) + .WithNoMapping()) + .AddCandidate( + targetKey: "location-1-key", + candidate: DataFixture.DefaultMappableLocationOption()))) + // Has deleted time periods that cannot be mapped + .WithHasDeletedTimePeriods(true); + + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ApplyAutoMappings(instanceId); + + var updatedMapping = await GetDataSetVersionMapping(nextVersion); + + // Is an exact match but as there are deleted time periods so needs to be a major update. + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); + } } public class ApplyAutoMappingsFiltersTests( @@ -1094,7 +1271,7 @@ public class ApplyAutoMappingsFiltersTests( : ApplyAutoMappingsTests(fixture) { [Fact] - public async Task PartiallyComplete() + public async Task PartiallyComplete_MajorUpdate() { var (instanceId, originalVersion, nextVersion) = await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); @@ -1204,7 +1381,7 @@ public async Task PartiallyComplete() } [Fact] - public async Task Complete_ExactMatch() + public async Task Complete_ExactMatch_MinorUpdate() { var (instanceId, originalVersion, nextVersion) = await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); @@ -1312,7 +1489,7 @@ public async Task Complete_ExactMatch() } [Fact] - public async Task Complete_AllSourcesMapped_OtherUnmappedCandidatesExist() + public async Task Complete_AllSourcesMapped_NewCandidatesExist_MinorUpdate() { var (instanceId, originalVersion, nextVersion) = await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); @@ -1389,7 +1566,7 @@ public async Task Complete_AllSourcesMapped_OtherUnmappedCandidatesExist() // and their child filter options with mapping type of AutoNone should not count towards // the calculation of the FilterMappingsComplete flag. [Fact] - public async Task Complete_SomeFiltersAutoNone() + public async Task Complete_SomeFiltersAutoNone_MajorUpdate() { var (instanceId, originalVersion, nextVersion) = await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); @@ -1476,6 +1653,111 @@ public async Task Complete_SomeFiltersAutoNone() // resulting in a major version update. await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); } + + [Fact] + public async Task Complete_HasDeletedIndicators_MajorUpdate() + { + var (instanceId, originalVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + DataSetVersionMapping mapping = DataFixture + .DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(originalVersion.Id) + .WithTargetDataSetVersionId(nextVersion.Id) + .WithFilterMappingPlan(DataFixture + .DefaultFilterMappingPlan() + .AddFilterMapping("filter-1-key", DataFixture + .DefaultFilterMapping() + .WithNoMapping() + .AddOptionMapping("filter-1-option-1-key", DataFixture + .DefaultFilterOptionMapping() + .WithNoMapping())) + .AddFilterCandidate("filter-1-key", DataFixture + .DefaultFilterMappingCandidate() + .AddOptionCandidate("filter-1-option-1-key", DataFixture + .DefaultMappableFilterOption()))) + // Has deleted indicators that cannot be mapped + .WithHasDeletedIndicators(true); + + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ApplyAutoMappings(instanceId); + + var updatedMapping = await GetDataSetVersionMapping(nextVersion); + + // Is an exact match but as there are deleted indicators so needs to be a major update. + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); + } + + [Fact] + public async Task Complete_HasDeletedGeographicLevels_MajorUpdate() + { + var (instanceId, originalVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + DataSetVersionMapping mapping = DataFixture + .DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(originalVersion.Id) + .WithTargetDataSetVersionId(nextVersion.Id) + .WithFilterMappingPlan(DataFixture + .DefaultFilterMappingPlan() + .AddFilterMapping("filter-1-key", DataFixture + .DefaultFilterMapping() + .WithNoMapping() + .AddOptionMapping("filter-1-option-1-key", DataFixture + .DefaultFilterOptionMapping() + .WithNoMapping())) + .AddFilterCandidate("filter-1-key", DataFixture + .DefaultFilterMappingCandidate() + .AddOptionCandidate("filter-1-option-1-key", DataFixture + .DefaultMappableFilterOption()))) + // Has deleted geographic levels that cannot be mapped + .WithHasDeletedGeographicLevels(true); + + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ApplyAutoMappings(instanceId); + + var updatedMapping = await GetDataSetVersionMapping(nextVersion); + + // Is an exact match but as there are deleted geographic levels so needs to be a major update. + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); + } + + [Fact] + public async Task Complete_HasDeletedTimePeriods_MajorUpdate() + { + var (instanceId, originalVersion, nextVersion) = + await CreateNextDataSetVersionAndDataFiles(Stage.PreviousStage()); + + DataSetVersionMapping mapping = DataFixture + .DefaultDataSetVersionMapping() + .WithSourceDataSetVersionId(originalVersion.Id) + .WithTargetDataSetVersionId(nextVersion.Id) + .WithFilterMappingPlan(DataFixture + .DefaultFilterMappingPlan() + .AddFilterMapping("filter-1-key", DataFixture + .DefaultFilterMapping() + .WithNoMapping() + .AddOptionMapping("filter-1-option-1-key", DataFixture + .DefaultFilterOptionMapping() + .WithNoMapping())) + .AddFilterCandidate("filter-1-key", DataFixture + .DefaultFilterMappingCandidate() + .AddOptionCandidate("filter-1-option-1-key", DataFixture + .DefaultMappableFilterOption()))) + // Has deleted time periods that cannot be mapped + .WithHasDeletedTimePeriods(true); + + await AddTestData(context => context.DataSetVersionMappings.Add(mapping)); + + await ApplyAutoMappings(instanceId); + + var updatedMapping = await GetDataSetVersionMapping(nextVersion); + + // Is an exact match but as there are deleted time periods so needs to be a major update. + await AssertCorrectDataSetVersionNumbers(updatedMapping, "2.0.0"); + } } public class CompleteNextDataSetVersionMappingsMappingProcessingTests( diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionMappingService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionMappingService.cs index 252b8b2b4a4..887857d82ba 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionMappingService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionMappingService.cs @@ -101,17 +101,17 @@ public async Task ApplyAutoMappings( Guid nextDataSetVersionId, CancellationToken cancellationToken = default) { - var mappings = await publicDataDbContext + var mapping = await publicDataDbContext .DataSetVersionMappings .Include(mapping => mapping.TargetDataSetVersion) .SingleAsync( mapping => mapping.TargetDataSetVersionId == nextDataSetVersionId, cancellationToken); - AutoMapLocations(mappings.LocationMappingPlan); - AutoMapFilters(mappings.FilterMappingPlan); + AutoMapLocations(mapping.LocationMappingPlan); + AutoMapFilters(mapping.FilterMappingPlan); - mappings.LocationMappingsComplete = !mappings.LocationMappingPlan + mapping.LocationMappingsComplete = !mapping.LocationMappingPlan .Levels // Ignore any levels where candidates or mappings are empty as this means the level // has been added or deleted from the data set and is not a mappable change. @@ -121,43 +121,64 @@ public async Task ApplyAutoMappings( // Note that currently within the UI there is no way to resolve unmapped filters, and therefore we // omit checking the status of filters that have a mapping of AutoNone. - mappings.FilterMappingsComplete = !mappings.FilterMappingPlan + mapping.FilterMappingsComplete = !mapping.FilterMappingPlan .Mappings .Where(filterMapping => filterMapping.Value.Type != MappingType.AutoNone) .SelectMany(filterMapping => filterMapping.Value.OptionMappings) .Any(optionMapping => IncompleteMappingTypes.Contains(optionMapping.Value.Type)); - var hasMajorLocationChange = mappings.LocationMappingPlan - .Levels - .Any(level => level.Value.Candidates.Count == 0 - || level.Value.Mappings - .Any(optionMapping => NoMappingTypes.Contains(optionMapping.Value.Type))); - - var hasMajorFilterUpdate = mappings.FilterMappingPlan - .Mappings - .SelectMany(filterMapping => filterMapping.Value.OptionMappings) - .Any(optionMapping => NoMappingTypes.Contains(optionMapping.Value.Type)); - - var isMajorVersionUpdate = hasMajorLocationChange || hasMajorFilterUpdate; - - if (isMajorVersionUpdate) + if (IsMajorVersionUpdate(mapping)) { - mappings.TargetDataSetVersion.VersionMajor += 1; - mappings.TargetDataSetVersion.VersionMinor = 0; + mapping.TargetDataSetVersion.VersionMajor += 1; + mapping.TargetDataSetVersion.VersionMinor = 0; } - publicDataDbContext.DataSetVersionMappings.Update(mappings); + publicDataDbContext.DataSetVersionMappings.Update(mapping); await publicDataDbContext.SaveChangesAsync(cancellationToken); var releaseFile = await contentDbContext.ReleaseFiles - .Where(rf => rf.Id == mappings.TargetDataSetVersion.Release.ReleaseFileId) + .Where(rf => rf.Id == mapping.TargetDataSetVersion.Release.ReleaseFileId) .SingleAsync(cancellationToken); - releaseFile.PublicApiDataSetVersion = mappings.TargetDataSetVersion.SemVersion(); + releaseFile.PublicApiDataSetVersion = mapping.TargetDataSetVersion.SemVersion(); await contentDbContext.SaveChangesAsync(cancellationToken); } + private static bool IsMajorVersionUpdate(DataSetVersionMapping mapping) + { + if (mapping.HasDeletedIndicators + || mapping.HasDeletedGeographicLevels + || mapping.HasDeletedTimePeriods) + { + return true; + } + + var hasDeletedLocationLevels = mapping.LocationMappingPlan + .Levels + .Any(level => level.Value.Candidates.Count == 0); + + if (hasDeletedLocationLevels) + { + return true; + } + + var hasUnmappedLocationOptions = mapping.LocationMappingPlan + .Levels + .Any(level => level.Value.Mappings + .Any(optionMapping => NoMappingTypes.Contains(optionMapping.Value.Type))); + + if (hasUnmappedLocationOptions) + { + return true; + } + + return mapping.FilterMappingPlan + .Mappings + .SelectMany(filterMapping => filterMapping.Value.OptionMappings) + .Any(optionMapping => NoMappingTypes.Contains(optionMapping.Value.Type)); + } + public Task>> GetManualMappingVersionAndImport( NextDataSetVersionCompleteImportRequest request, CancellationToken cancellationToken = default) From 8fab39f0599a73207d3548a3dc52cbfa43248e01 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Mon, 14 Oct 2024 12:17:57 +0100 Subject: [PATCH 65/80] EES-5572 Remove frontend ordering of changelog entries This defers ordering to the backend, which is done upon generating the changelog in the public processor's `DataSetVersionChangeService`. --- .../components/ChangeSection.tsx | 69 +++----------- .../__tests__/ChangeSection.test.tsx | 92 +++++++++---------- 2 files changed, 58 insertions(+), 103 deletions(-) diff --git a/src/explore-education-statistics-common/src/modules/data-catalogue/components/ChangeSection.tsx b/src/explore-education-statistics-common/src/modules/data-catalogue/components/ChangeSection.tsx index d08dfa96a9e..0789db1ac4e 100644 --- a/src/explore-education-statistics-common/src/modules/data-catalogue/components/ChangeSection.tsx +++ b/src/explore-education-statistics-common/src/modules/data-catalogue/components/ChangeSection.tsx @@ -1,7 +1,5 @@ import ChangeList from '@common/modules/data-catalogue/components/ChangeList'; import { ChangeSet } from '@common/services/types/apiDataSetChanges'; -import naturalOrderBy from '@common/utils/array/naturalOrderBy'; -import orderBy from 'lodash/orderBy'; import React from 'react'; interface Props { @@ -9,57 +7,15 @@ interface Props { } export default function ChangeSection({ changes }: Props) { - const filters = naturalOrderBy( - changes.filters ?? [], - filter => filter.previousState?.label ?? filter.currentState?.label ?? '', - ); - - const filterOptions = naturalOrderBy( - changes.filterOptions ?? [], - group => group.filter.label, - ); - - const geographicLevels = orderBy( - changes.geographicLevels ?? [], - level => level.previousState?.label ?? level.currentState?.label, - ); - - const indicators = naturalOrderBy( - changes.indicators ?? [], - indicator => - indicator.previousState?.label ?? indicator.currentState?.label ?? '', - ); - - const locationGroups = orderBy( - changes.locationGroups ?? [], - group => - group.previousState?.level.label ?? group.currentState?.level.label, - ); - - const locationOptions = orderBy( - changes.locationOptions ?? [], - group => group.level.label, - ); - - const timePeriods = naturalOrderBy( - changes.timePeriods ?? [], - timePeriod => - timePeriod.previousState?.label ?? timePeriod.currentState?.label ?? '', - ); - return ( <> - + - {filterOptions.map(group => { + {changes.filterOptions?.map(group => { return ( - option.previousState?.label ?? option?.currentState?.label, - )} + changes={group.options} metaType="filterOptions" metaTypeLabel={`${group.filter.label} filter options`} testId={`filterOptions-${group.filter.id}`} @@ -67,21 +23,20 @@ export default function ChangeSection({ changes }: Props) { ); })} - + - + - + - {locationOptions.map(group => { + {changes.locationOptions?.map(group => { return ( - option.previousState?.label ?? option?.currentState?.label, - )} + changes={group.options} metaType="locationOptions" metaTypeLabel={`${group.level.label} location options`} testId={`locationOptions-${group.level.code}`} @@ -89,7 +44,7 @@ export default function ChangeSection({ changes }: Props) { ); })} - + ); } diff --git a/src/explore-education-statistics-common/src/modules/data-catalogue/components/__tests__/ChangeSection.test.tsx b/src/explore-education-statistics-common/src/modules/data-catalogue/components/__tests__/ChangeSection.test.tsx index ed84361c46b..4dad0676f98 100644 --- a/src/explore-education-statistics-common/src/modules/data-catalogue/components/__tests__/ChangeSection.test.tsx +++ b/src/explore-education-statistics-common/src/modules/data-catalogue/components/__tests__/ChangeSection.test.tsx @@ -7,8 +7,12 @@ describe('ChangeSection', () => { test('renders filter changes only', () => { const testChanges: ChangeSet = { filters: [ - { previousState: { id: 'filter-2', label: 'Filter 2', hint: '' } }, { previousState: { id: 'filter-1', label: 'Filter 1', hint: '' } }, + { previousState: { id: 'filter-2', label: 'Filter 2', hint: '' } }, + { + previousState: { id: 'filter-3', label: 'Filter 3', hint: '' }, + currentState: { id: 'filter-3-updated', label: 'Filter 3', hint: '' }, + }, { previousState: { id: 'filter-4', @@ -21,12 +25,8 @@ describe('ChangeSection', () => { hint: 'Filter 4 hint updated', }, }, - { - previousState: { id: 'filter-3', label: 'Filter 3', hint: '' }, - currentState: { id: 'filter-3-updated', label: 'Filter 3', hint: '' }, - }, - { currentState: { id: 'filter-6', label: 'Filter 6', hint: '' } }, { currentState: { id: 'filter-5', label: 'Filter 5', hint: '' } }, + { currentState: { id: 'filter-6', label: 'Filter 6', hint: '' } }, ], }; @@ -71,25 +71,25 @@ describe('ChangeSection', () => { { filter: { id: 'filter-1', label: 'Filter 1', hint: '' }, options: [ - { previousState: { id: 'filter-opt-2', label: 'Filter option 2' } }, { previousState: { id: 'filter-opt-1', label: 'Filter option 1' } }, + { previousState: { id: 'filter-opt-2', label: 'Filter option 2' } }, { - previousState: { id: 'filter-opt-4', label: 'Filter option 4' }, + previousState: { id: 'filter-opt-3', label: 'Filter option 3' }, currentState: { - id: 'filter-opt-4', - label: 'Filter option 4 updated', - isAggregate: true, + id: 'filter-opt-3-updated', + label: 'Filter option 3', }, }, { - previousState: { id: 'filter-opt-3', label: 'Filter option 3' }, + previousState: { id: 'filter-opt-4', label: 'Filter option 4' }, currentState: { - id: 'filter-opt-3-updated', - label: 'Filter option 3', + id: 'filter-opt-4', + label: 'Filter option 4 updated', + isAggregate: true, }, }, - { currentState: { id: 'filter-opt-6', label: 'Filter option 6' } }, { currentState: { id: 'filter-opt-5', label: 'Filter option 5' } }, + { currentState: { id: 'filter-opt-6', label: 'Filter option 6' } }, ], }, { @@ -197,18 +197,18 @@ describe('ChangeSection', () => { test('renders geographic level changes only', () => { const testChanges: ChangeSet = { geographicLevels: [ - { previousState: { code: 'WARD', label: 'Ward' } }, { previousState: { code: 'NAT', label: 'National' } }, - { - previousState: { code: 'OA', label: 'Opportunity area' }, - currentState: { code: 'PA', label: 'Planning area' }, - }, + { previousState: { code: 'WARD', label: 'Ward' } }, { previousState: { code: 'LA', label: 'Local authority' }, currentState: { code: 'LAD', label: 'Local authority district' }, }, - { currentState: { code: 'REG', label: 'Regional' } }, + { + previousState: { code: 'OA', label: 'Opportunity area' }, + currentState: { code: 'PA', label: 'Planning area' }, + }, { currentState: { code: 'NAT', label: 'National' } }, + { currentState: { code: 'REG', label: 'Regional' } }, ], }; @@ -254,8 +254,8 @@ describe('ChangeSection', () => { test('renders indicators changes only', () => { const testChanges: ChangeSet = { indicators: [ - { previousState: { id: 'indicator-2', label: 'Indicator 2' } }, { previousState: { id: 'indicator-1', label: 'Indicator 1' } }, + { previousState: { id: 'indicator-2', label: 'Indicator 2' } }, { currentState: { id: 'indicator-4', label: 'Indicator 3' }, previousState: { id: 'indicator-3', label: 'Indicator 3' }, @@ -268,8 +268,8 @@ describe('ChangeSection', () => { }, previousState: { id: 'indicator-5', label: 'Indicator 5' }, }, - { currentState: { id: 'indicator-7', label: 'Indicator 7' } }, { currentState: { id: 'indicator-6', label: 'Indicator 6' } }, + { currentState: { id: 'indicator-7', label: 'Indicator 7' } }, ], }; @@ -313,20 +313,20 @@ describe('ChangeSection', () => { test('renders location group changes only', () => { const testChanges: ChangeSet = { locationGroups: [ - { previousState: { level: { code: 'WARD', label: 'Ward' } } }, { previousState: { level: { code: 'INST', label: 'Institution' } } }, - { - previousState: { level: { code: 'OA', label: 'Opportunity area' } }, - currentState: { level: { code: 'PA', label: 'Planning area' } }, - }, + { previousState: { level: { code: 'WARD', label: 'Ward' } } }, { previousState: { level: { code: 'LA', label: 'Local authority' } }, currentState: { level: { code: 'LAD', label: 'Local authority district' }, }, }, - { currentState: { level: { code: 'REG', label: 'Regional' } } }, + { + previousState: { level: { code: 'OA', label: 'Opportunity area' } }, + currentState: { level: { code: 'PA', label: 'Planning area' } }, + }, { currentState: { level: { code: 'NAT', label: 'National' } } }, + { currentState: { level: { code: 'REG', label: 'Regional' } } }, ], }; @@ -377,13 +377,6 @@ describe('ChangeSection', () => { { level: { code: 'REG', label: 'Regional' }, options: [ - { - previousState: { - id: 'location-2', - code: 'location-2-code', - label: 'Location 2', - }, - }, { previousState: { id: 'location-1', @@ -393,14 +386,9 @@ describe('ChangeSection', () => { }, { previousState: { - id: 'location-4', - label: 'Location 4', - ukprn: 'location-4-ukprn', - }, - currentState: { - id: 'location-4', - label: 'Location 4 updated', - ukprn: 'location-4-ukprn-updated', + id: 'location-2', + code: 'location-2-code', + label: 'Location 2', }, }, { @@ -417,6 +405,18 @@ describe('ChangeSection', () => { oldCode: 'location-3-oldCode-updated', }, }, + { + previousState: { + id: 'location-4', + label: 'Location 4', + ukprn: 'location-4-ukprn', + }, + currentState: { + id: 'location-4', + label: 'Location 4 updated', + ukprn: 'location-4-ukprn-updated', + }, + }, { currentState: { id: 'location-5', @@ -582,10 +582,10 @@ describe('ChangeSection', () => { const testChanges: ChangeSet = { timePeriods: [ { - previousState: { code: 'AY', label: '2018/19', period: '2018/2019' }, + previousState: { code: 'AY', label: '2017/18', period: '2017/2018' }, }, { - previousState: { code: 'AY', label: '2017/18', period: '2017/2018' }, + previousState: { code: 'AY', label: '2018/19', period: '2018/2019' }, }, { previousState: { code: 'AY', label: '2019/20', period: '2019/2020' }, From 655f17df0d52b6e1da58ced7c9a7baf734dd3df5 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Mon, 14 Oct 2024 14:46:24 +0100 Subject: [PATCH 66/80] EES-5572 Add column names for filters and indicators in changelog --- .../ReleaseApiDataSetChangelogPage.test.tsx | 20 +- .../components/FilterChangeLabel.tsx | 11 +- .../components/IndicatorChangeLabel.tsx | 8 +- .../__tests__/ApiDataSetChangelog.test.tsx | 52 ++- .../__tests__/ChangeSection.test.tsx | 344 +++++++++++++++--- .../src/services/types/apiDataSetMeta.ts | 2 + .../__tests__/DataSetFilePage.test.tsx | 18 +- .../DataSetFileApiChangelog.test.tsx | 43 ++- 8 files changed, 413 insertions(+), 85 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetChangelogPage.test.tsx b/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetChangelogPage.test.tsx index 78b614f4ff7..b17bd2c5a11 100644 --- a/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetChangelogPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/release/data/__tests__/ReleaseApiDataSetChangelogPage.test.tsx @@ -71,14 +71,24 @@ describe('ReleaseApiDataSetChangelogPage', () => { majorChanges: { filters: [ { - previousState: { id: 'filter-1', label: 'Filter 1', hint: '' }, + previousState: { + id: 'filter-1', + column: 'filter_1', + label: 'Filter 1', + hint: '', + }, }, ], }, minorChanges: { filters: [ { - currentState: { id: 'filter-2', label: 'Filter 2', hint: '' }, + currentState: { + id: 'filter-2', + column: 'filter_2', + label: 'Filter 2', + hint: '', + }, }, ], }, @@ -115,7 +125,7 @@ describe('ReleaseApiDataSetChangelogPage', () => { const minorChanges = within(screen.getByTestId('minor-changes')); expect(minorChanges.getByTestId('added-filters')).toHaveTextContent( - 'Filter 2 (id: filter-2)', + 'Filter 2 (id: filter-2, column: filter_2)', ); }); @@ -142,7 +152,7 @@ describe('ReleaseApiDataSetChangelogPage', () => { const majorChanges = within(screen.getByTestId('major-changes')); expect(majorChanges.getByTestId('deleted-filters')).toHaveTextContent( - 'Filter 1 (id: filter-1)', + 'Filter 1 (id: filter-1, column: filter_1)', ); expect( @@ -152,7 +162,7 @@ describe('ReleaseApiDataSetChangelogPage', () => { const minorChanges = within(screen.getByTestId('minor-changes')); expect(minorChanges.getByTestId('added-filters')).toHaveTextContent( - 'Filter 2 (id: filter-2)', + 'Filter 2 (id: filter-2, column: filter_2)', ); }); diff --git a/src/explore-education-statistics-common/src/modules/data-catalogue/components/FilterChangeLabel.tsx b/src/explore-education-statistics-common/src/modules/data-catalogue/components/FilterChangeLabel.tsx index 0d6e3dfd37f..7741c954646 100644 --- a/src/explore-education-statistics-common/src/modules/data-catalogue/components/FilterChangeLabel.tsx +++ b/src/explore-education-statistics-common/src/modules/data-catalogue/components/FilterChangeLabel.tsx @@ -13,7 +13,8 @@ export default function FilterChangeLabel({ if (previousState && currentState) { return ( <> - {previousState.label} (id: {previousState.id}): + {previousState.label} (id: {previousState.id}, column:{' '} + {previousState.column}):
      {previousState.label !== currentState.label && (
    • label changed to: {currentState.label}
      • @@ -23,6 +24,11 @@ export default function FilterChangeLabel({ id changed to: {currentState.id} )} + {previousState.column !== currentState.column && ( +
    • + column changed to: {currentState.column} +
      • + )} {previousState.hint !== currentState.hint && (
    • hint changed to: {currentState.hint}
      • )} @@ -39,7 +45,8 @@ export default function FilterChangeLabel({ return ( <> - {state.label} (id: {state.id}) + {state.label} (id: {state.id}, column:{' '} + {state.column}) ); } diff --git a/src/explore-education-statistics-common/src/modules/data-catalogue/components/IndicatorChangeLabel.tsx b/src/explore-education-statistics-common/src/modules/data-catalogue/components/IndicatorChangeLabel.tsx index cfe4e491115..2ef0cb975f1 100644 --- a/src/explore-education-statistics-common/src/modules/data-catalogue/components/IndicatorChangeLabel.tsx +++ b/src/explore-education-statistics-common/src/modules/data-catalogue/components/IndicatorChangeLabel.tsx @@ -13,7 +13,8 @@ export default function IndicatorChangeLabel({ if (previousState && currentState) { return ( <> - {previousState.label} (id: {previousState.id}): + {previousState.label} (id: {previousState.id}, column:{' '} + {previousState.column}):
        {previousState.label !== currentState.label && (
      • label changed to: {currentState.label}
        • @@ -23,6 +24,11 @@ export default function IndicatorChangeLabel({ id changed to: {currentState.id} )} + {previousState.column !== currentState.column && ( +
      • + column changed to: {currentState.column} +
        • + )} {previousState.unit !== currentState.unit && (
      • unit changed to: {currentState.unit} diff --git a/src/explore-education-statistics-common/src/modules/data-catalogue/components/__tests__/ApiDataSetChangelog.test.tsx b/src/explore-education-statistics-common/src/modules/data-catalogue/components/__tests__/ApiDataSetChangelog.test.tsx index bdd57490571..dbb820ff30b 100644 --- a/src/explore-education-statistics-common/src/modules/data-catalogue/components/__tests__/ApiDataSetChangelog.test.tsx +++ b/src/explore-education-statistics-common/src/modules/data-catalogue/components/__tests__/ApiDataSetChangelog.test.tsx @@ -9,7 +9,14 @@ describe('ApiDataSetChangelog', () => { version="2.0" majorChanges={{ filters: [ - { previousState: { id: 'filter-1', label: 'Filter 1', hint: '' } }, + { + previousState: { + id: 'filter-1', + column: 'filter_1', + label: 'Filter 1', + hint: '', + }, + }, ], }} minorChanges={{}} @@ -33,7 +40,9 @@ describe('ApiDataSetChangelog', () => { ).getAllByRole('listitem'); expect(deletedFilters).toHaveLength(1); - expect(deletedFilters[0]).toHaveTextContent('Filter 1 (id: filter-1)'); + expect(deletedFilters[0]).toHaveTextContent( + 'Filter 1 (id: filter-1, column: filter_1)', + ); // No minor changes expect(screen.queryByTestId('minor-changes')).not.toBeInTheDocument(); @@ -46,7 +55,14 @@ describe('ApiDataSetChangelog', () => { majorChanges={{}} minorChanges={{ filters: [ - { currentState: { id: 'filter-2', label: 'Filter 2', hint: '' } }, + { + currentState: { + id: 'filter-2', + column: 'filter_2', + label: 'Filter 2', + hint: '', + }, + }, ], }} />, @@ -73,7 +89,9 @@ describe('ApiDataSetChangelog', () => { expect(addedFilters).toHaveLength(1); - expect(addedFilters[0]).toHaveTextContent('Filter 2 (id: filter-2)'); + expect(addedFilters[0]).toHaveTextContent( + 'Filter 2 (id: filter-2, column: filter_2)', + ); }); test('renders major and minor changes', () => { @@ -82,12 +100,26 @@ describe('ApiDataSetChangelog', () => { version="2.0" majorChanges={{ filters: [ - { previousState: { id: 'filter-1', label: 'Filter 1', hint: '' } }, + { + previousState: { + id: 'filter-1', + column: 'filter_1', + label: 'Filter 1', + hint: '', + }, + }, ], }} minorChanges={{ filters: [ - { currentState: { id: 'filter-2', label: 'Filter 2', hint: '' } }, + { + currentState: { + id: 'filter-2', + column: 'filter_2', + label: 'Filter 2', + hint: '', + }, + }, ], }} />, @@ -110,7 +142,9 @@ describe('ApiDataSetChangelog', () => { ).getAllByRole('listitem'); expect(deletedFilters).toHaveLength(1); - expect(deletedFilters[0]).toHaveTextContent('Filter 1 (id: filter-1)'); + expect(deletedFilters[0]).toHaveTextContent( + 'Filter 1 (id: filter-1, column: filter_1)', + ); const minorChanges = within(screen.getByTestId('minor-changes')); @@ -129,6 +163,8 @@ describe('ApiDataSetChangelog', () => { ).getAllByRole('listitem'); expect(addedFilters).toHaveLength(1); - expect(addedFilters[0]).toHaveTextContent('Filter 2 (id: filter-2)'); + expect(addedFilters[0]).toHaveTextContent( + 'Filter 2 (id: filter-2, column: filter_2)', + ); }); }); diff --git a/src/explore-education-statistics-common/src/modules/data-catalogue/components/__tests__/ChangeSection.test.tsx b/src/explore-education-statistics-common/src/modules/data-catalogue/components/__tests__/ChangeSection.test.tsx index 4dad0676f98..45b138d1648 100644 --- a/src/explore-education-statistics-common/src/modules/data-catalogue/components/__tests__/ChangeSection.test.tsx +++ b/src/explore-education-statistics-common/src/modules/data-catalogue/components/__tests__/ChangeSection.test.tsx @@ -7,26 +7,66 @@ describe('ChangeSection', () => { test('renders filter changes only', () => { const testChanges: ChangeSet = { filters: [ - { previousState: { id: 'filter-1', label: 'Filter 1', hint: '' } }, - { previousState: { id: 'filter-2', label: 'Filter 2', hint: '' } }, { - previousState: { id: 'filter-3', label: 'Filter 3', hint: '' }, - currentState: { id: 'filter-3-updated', label: 'Filter 3', hint: '' }, + previousState: { + id: 'filter-1', + column: 'filter_1', + label: 'Filter 1', + hint: '', + }, + }, + { + previousState: { + id: 'filter-2', + column: 'filter_2', + label: 'Filter 2', + hint: '', + }, + }, + { + previousState: { + id: 'filter-3', + column: 'filter_3', + label: 'Filter 3', + hint: '', + }, + currentState: { + id: 'filter-3-updated', + column: 'filter_3_updated', + label: 'Filter 3', + hint: '', + }, }, { previousState: { id: 'filter-4', + column: 'filter_4', label: 'Filter 4', hint: 'Filter 4 hint', }, currentState: { id: 'filter-4', + column: 'filter_4', label: 'Filter 4 updated', hint: 'Filter 4 hint updated', }, }, - { currentState: { id: 'filter-5', label: 'Filter 5', hint: '' } }, - { currentState: { id: 'filter-6', label: 'Filter 6', hint: '' } }, + { + currentState: { + id: 'filter-5', + column: 'filter_5', + label: 'Filter 5', + hint: '', + }, + }, + { + currentState: { + id: 'filter-6', + column: 'filter_6', + label: 'Filter 6', + hint: '', + }, + }, ], }; @@ -41,35 +81,69 @@ describe('ChangeSection', () => { ); expect(deleted).toHaveLength(2); - expect(deleted[0]).toHaveTextContent('Filter 1 (id: filter-1)'); - expect(deleted[1]).toHaveTextContent('Filter 2 (id: filter-2)'); + expect(deleted[0]).toHaveTextContent( + 'Filter 1 (id: filter-1, column: filter_1)', + ); + expect(deleted[1]).toHaveTextContent( + 'Filter 2 (id: filter-2, column: filter_2)', + ); const updated = within( screen.getByTestId('updated-filters'), ).getAllByTestId('updated-item'); expect(updated).toHaveLength(2); - expect(updated[0]).toHaveTextContent('Filter 3 (id: filter-3)'); - expect(updated[0]).toHaveTextContent('id changed to: filter-3-updated'); + expect(updated[0]).toHaveTextContent( + 'Filter 3 (id: filter-3, column: filter_3)', + ); - expect(updated[1]).toHaveTextContent('Filter 4 (id: filter-4)'); - expect(updated[1]).toHaveTextContent('label changed to: Filter 4 updated'); - expect(updated[1]).toHaveTextContent('hint changed to: Filter 4 hint'); + const updated1Changes = within(updated[0]).getAllByRole('listitem'); + + expect(updated1Changes).toHaveLength(2); + expect(updated1Changes[0]).toHaveTextContent( + 'id changed to: filter-3-updated', + ); + expect(updated1Changes[1]).toHaveTextContent( + 'column changed to: filter_3_updated', + ); + + expect(updated[1]).toHaveTextContent( + 'Filter 4 (id: filter-4, column: filter_4)', + ); + + const updated2Changes = within(updated[1]).getAllByRole('listitem'); + + expect(updated2Changes).toHaveLength(2); + expect(updated2Changes[0]).toHaveTextContent( + 'label changed to: Filter 4 updated', + ); + expect(updated2Changes[1]).toHaveTextContent( + 'hint changed to: Filter 4 hint', + ); const added = within(screen.getByTestId('added-filters')).getAllByRole( 'listitem', ); expect(added).toHaveLength(2); - expect(added[0]).toHaveTextContent('Filter 5 (id: filter-5)'); - expect(added[1]).toHaveTextContent('Filter 6 (id: filter-6)'); + expect(added[0]).toHaveTextContent( + 'Filter 5 (id: filter-5, column: filter_5)', + ); + expect(added[1]).toHaveTextContent( + 'Filter 6 (id: filter-6, column: filter_6)', + ); }); test('renders filter option changes only', () => { const testChanges: ChangeSet = { filterOptions: [ { - filter: { id: 'filter-1', label: 'Filter 1', hint: '' }, + filter: { + id: 'filter-1', + column: 'filter_1', + label: 'Filter 1', + hint: '', + }, options: [ { previousState: { id: 'filter-opt-1', label: 'Filter option 1' } }, { previousState: { id: 'filter-opt-2', label: 'Filter option 2' } }, @@ -93,7 +167,12 @@ describe('ChangeSection', () => { ], }, { - filter: { id: 'filter-2', label: 'Filter 2', hint: '' }, + filter: { + id: 'filter-2', + column: 'filter_2', + label: 'Filter 2', + hint: '', + }, options: [ { previousState: { id: 'filter-opt-7', label: 'Filter option 7' } }, { @@ -138,16 +217,29 @@ describe('ChangeSection', () => { expect(updatedFilter1Options[0]).toHaveTextContent( 'Filter option 3 (id: filter-opt-3)', ); - expect(updatedFilter1Options[0]).toHaveTextContent( + + const updatedFilter1Option1Changes = within( + updatedFilter1Options[0], + ).getAllByRole('listitem'); + + expect(updatedFilter1Option1Changes).toHaveLength(1); + expect(updatedFilter1Option1Changes[0]).toHaveTextContent( 'id changed to: filter-opt-3-updated', ); + expect(updatedFilter1Options[1]).toHaveTextContent( 'Filter option 4 (id: filter-opt-4)', ); - expect(updatedFilter1Options[1]).toHaveTextContent( + + const updatedFilter1Option2Changes = within( + updatedFilter1Options[1], + ).getAllByRole('listitem'); + + expect(updatedFilter1Option2Changes).toHaveLength(2); + expect(updatedFilter1Option2Changes[0]).toHaveTextContent( 'label changed to: Filter option 4 updated', ); - expect(updatedFilter1Options[1]).toHaveTextContent( + expect(updatedFilter1Option2Changes[1]).toHaveTextContent( 'changed to an aggregate', ); @@ -180,6 +272,12 @@ describe('ChangeSection', () => { expect(updatedFilter2Options[0]).toHaveTextContent( 'Filter option 8 (id: filter-opt-8)', ); + + const updatedFilter2Option1Changes = within( + updatedFilter2Options[0], + ).getAllByRole('listitem'); + + expect(updatedFilter2Option1Changes).toHaveLength(1); expect(updatedFilter2Options[0]).toHaveTextContent( 'no longer an aggregate', ); @@ -234,11 +332,20 @@ describe('ChangeSection', () => { expect(updated).toHaveLength(2); expect(updated[0]).toHaveTextContent('Local authority (code: LA)'); - expect(updated[0]).toHaveTextContent( + + const updated1Changes = within(updated[0]).getAllByRole('listitem'); + + expect(updated1Changes).toHaveLength(1); + expect(updated1Changes[0]).toHaveTextContent( 'changed to: Local authority district (code: LAD)', ); + expect(updated[1]).toHaveTextContent('Opportunity area (code: OA)'); - expect(updated[1]).toHaveTextContent( + + const updated2Changes = within(updated[1]).getAllByRole('listitem'); + + expect(updated2Changes).toHaveLength(1); + expect(updated2Changes[0]).toHaveTextContent( 'changed to: Planning area (code: PA)', ); @@ -254,22 +361,59 @@ describe('ChangeSection', () => { test('renders indicators changes only', () => { const testChanges: ChangeSet = { indicators: [ - { previousState: { id: 'indicator-1', label: 'Indicator 1' } }, - { previousState: { id: 'indicator-2', label: 'Indicator 2' } }, { - currentState: { id: 'indicator-4', label: 'Indicator 3' }, - previousState: { id: 'indicator-3', label: 'Indicator 3' }, + previousState: { + id: 'indicator-1', + column: 'indicator_1', + label: 'Indicator 1', + }, + }, + { + previousState: { + id: 'indicator-2', + column: 'indicator_2', + label: 'Indicator 2', + }, + }, + { + currentState: { + id: 'indicator-3-updated', + column: 'indicator_3_updated', + label: 'Indicator 3', + }, + previousState: { + id: 'indicator-3', + column: 'indicator_3', + label: 'Indicator 3', + }, }, { currentState: { - id: 'indicator-5', - label: 'Indicator 5 updated', + id: 'indicator-4', + column: 'indicator_4', + label: 'Indicator 4 updated', unit: '%', }, - previousState: { id: 'indicator-5', label: 'Indicator 5' }, + previousState: { + id: 'indicator-4', + column: 'indicator_4', + label: 'Indicator 4', + }, + }, + { + currentState: { + id: 'indicator-6', + column: 'indicator_6', + label: 'Indicator 6', + }, + }, + { + currentState: { + id: 'indicator-7', + column: 'indicator_7', + label: 'Indicator 7', + }, }, - { currentState: { id: 'indicator-6', label: 'Indicator 6' } }, - { currentState: { id: 'indicator-7', label: 'Indicator 7' } }, ], }; @@ -292,14 +436,31 @@ describe('ChangeSection', () => { ).getAllByTestId('updated-item'); expect(updated).toHaveLength(2); - expect(updated[0]).toHaveTextContent('Indicator 3 (id: indicator-3)'); - expect(updated[0]).toHaveTextContent('id changed to: indicator-4'); + expect(updated[0]).toHaveTextContent( + 'Indicator 3 (id: indicator-3, column: indicator_3)', + ); + + const updated1Changes = within(updated[0]).getAllByRole('listitem'); + + expect(updated1Changes).toHaveLength(2); + expect(updated1Changes[0]).toHaveTextContent( + 'id changed to: indicator-3-updated', + ); + expect(updated1Changes[1]).toHaveTextContent( + 'column changed to: indicator_3_updated', + ); - expect(updated[1]).toHaveTextContent('Indicator 5 (id: indicator-5)'); expect(updated[1]).toHaveTextContent( - 'label changed to: Indicator 5 updated', + 'Indicator 4 (id: indicator-4, column: indicator_4)', ); - expect(updated[1]).toHaveTextContent('unit changed to: %'); + + const updated2Changes = within(updated[1]).getAllByRole('listitem'); + + expect(updated2Changes).toHaveLength(2); + expect(updated2Changes[0]).toHaveTextContent( + 'label changed to: Indicator 4 updated', + ); + expect(updated2Changes[1]).toHaveTextContent('unit changed to: %'); const added = within(screen.getByTestId('added-indicators')).getAllByRole( 'listitem', @@ -352,12 +513,20 @@ describe('ChangeSection', () => { expect(updated).toHaveLength(2); expect(updated[0]).toHaveTextContent('Local authority (code: LA)'); - expect(updated[0]).toHaveTextContent( + + const updated1Changes = within(updated[0]).getAllByRole('listitem'); + + expect(updated1Changes).toHaveLength(1); + expect(updated1Changes[0]).toHaveTextContent( 'changed to: Local authority district (code: LAD)', ); expect(updated[1]).toHaveTextContent('Opportunity area (code: OA)'); - expect(updated[1]).toHaveTextContent( + + const updated2Changes = within(updated[1]).getAllByRole('listitem'); + + expect(updated2Changes).toHaveLength(1); + expect(updated2Changes[0]).toHaveTextContent( 'changed to: Planning area (code: PA)', ); @@ -499,24 +668,36 @@ describe('ChangeSection', () => { expect(updatedRegionOptions[0]).toHaveTextContent( 'Location 3 (id: location-3, code: location-3-code, old code: location-3-oldCode)', ); - expect(updatedRegionOptions[0]).toHaveTextContent( + + const updatedRegionOption1Changes = within( + updatedRegionOptions[0], + ).getAllByRole('listitem'); + + expect(updatedRegionOption1Changes).toHaveLength(3); + expect(updatedRegionOption1Changes[0]).toHaveTextContent( 'id changed to: location-3-updated', ); - expect(updatedRegionOptions[0]).toHaveTextContent( + expect(updatedRegionOption1Changes[1]).toHaveTextContent( 'code changed to: location-3-code-updated', ); - expect(updatedRegionOptions[0]).toHaveTextContent( + expect(updatedRegionOption1Changes[2]).toHaveTextContent( 'old code changed to: location-3-oldCode-updated', ); expect(updatedRegionOptions[1]).toHaveTextContent( 'Location 4 (id: location-4, UKPRN: location-4-ukprn)', ); - expect(updatedRegionOptions[1]).toHaveTextContent( + + const updatedRegionOption2Changes = within( + updatedRegionOptions[1], + ).getAllByRole('listitem'); + + expect(updatedRegionOption2Changes).toHaveLength(2); + expect(updatedRegionOption2Changes[0]).toHaveTextContent( 'label changed to: Location 4 updated', ); - expect(updatedRegionOptions[1]).toHaveTextContent( - 'UKPRN changed to: location-4-ukprn', + expect(updatedRegionOption2Changes[1]).toHaveTextContent( + 'UKPRN changed to: location-4-ukprn-updated', ); const addedRegionOptions = within( @@ -555,16 +736,22 @@ describe('ChangeSection', () => { expect(updatedSchoolOptions[0]).toHaveTextContent( 'Location 8 (id: location-8, URN: location-8-urn, LAESTAB: location-8-laEstab)', ); - expect(updatedSchoolOptions[0]).toHaveTextContent( - 'id changed to: location-8-updated', - ); - expect(updatedSchoolOptions[0]).toHaveTextContent( + + const updatedSchoolOption1Changes = within( + updatedSchoolOptions[0], + ).getAllByRole('listitem'); + + expect(updatedSchoolOption1Changes).toHaveLength(4); + expect(updatedSchoolOption1Changes[0]).toHaveTextContent( 'label changed to: Location 8 updated', ); - expect(updatedSchoolOptions[0]).toHaveTextContent( + expect(updatedSchoolOption1Changes[1]).toHaveTextContent( + 'id changed to: location-8-updated', + ); + expect(updatedSchoolOption1Changes[2]).toHaveTextContent( 'URN changed to: location-8-urn-updated', ); - expect(updatedSchoolOptions[0]).toHaveTextContent( + expect(updatedSchoolOption1Changes[3]).toHaveTextContent( 'LAESTAB changed to: location-8-laEstab-updated', ); @@ -640,13 +827,28 @@ describe('ChangeSection', () => { const testChanges: ChangeSet = { filters: [ { - previousState: { id: 'filter-1', label: 'Filter 1', hint: '' }, - currentState: { id: 'filter-1', label: 'Filter 1 updated', hint: '' }, + previousState: { + id: 'filter-1', + column: 'filter_1', + label: 'Filter 1', + hint: '', + }, + currentState: { + id: 'filter-1', + column: 'filter_1_updated', + label: 'Filter 1 updated', + hint: '', + }, }, ], filterOptions: [ { - filter: { id: 'filter-1', label: 'Filter 1', hint: '' }, + filter: { + id: 'filter-1', + column: 'filter_1', + label: 'Filter 1', + hint: '', + }, options: [ { previousState: { id: 'filter-opt-1', label: 'Filter option 1' } }, ], @@ -657,8 +859,16 @@ describe('ChangeSection', () => { ], indicators: [ { - currentState: { id: 'indicator-1', label: 'Indicator 1 updated' }, - previousState: { id: 'indicator-1', label: 'Indicator 1' }, + currentState: { + id: 'indicator-1', + column: 'indicator_1_updated', + label: 'Indicator 1 updated', + }, + previousState: { + id: 'indicator-1', + column: 'indicator_1', + label: 'Indicator 1', + }, }, ], locationGroups: [ @@ -697,10 +907,20 @@ describe('ChangeSection', () => { expect(updatedFilters).toHaveLength(1); - expect(updatedFilters[0]).toHaveTextContent('Filter 1 (id: filter-1)'); expect(updatedFilters[0]).toHaveTextContent( + 'Filter 1 (id: filter-1, column: filter_1)', + ); + + const updatedFilterChanges = within(updatedFilters[0]).getAllByRole( + 'listitem', + ); + expect(updatedFilterChanges).toHaveLength(2); + expect(updatedFilterChanges[0]).toHaveTextContent( 'label changed to: Filter 1 updated', ); + expect(updatedFilterChanges[1]).toHaveTextContent( + 'column changed to: filter_1_updated', + ); expect( screen.getByRole('heading', { name: 'Deleted Filter 1 filter options' }), @@ -738,11 +958,19 @@ describe('ChangeSection', () => { expect(updatedIndicators).toHaveLength(1); expect(updatedIndicators[0]).toHaveTextContent( - 'Indicator 1 (id: indicator-1)', + 'Indicator 1 (id: indicator-1, column: indicator_1)', ); - expect(updatedIndicators[0]).toHaveTextContent( + + const updatedIndicatorChanges = within(updatedIndicators[0]).getAllByRole( + 'listitem', + ); + expect(updatedIndicatorChanges).toHaveLength(2); + expect(updatedIndicatorChanges[0]).toHaveTextContent( 'label changed to: Indicator 1 updated', ); + expect(updatedIndicatorChanges[1]).toHaveTextContent( + 'column changed to: indicator_1_updated', + ); expect( screen.getByRole('heading', { name: 'New location groups' }), diff --git a/src/explore-education-statistics-common/src/services/types/apiDataSetMeta.ts b/src/explore-education-statistics-common/src/services/types/apiDataSetMeta.ts index 1cb741e1e62..1ffbf812609 100644 --- a/src/explore-education-statistics-common/src/services/types/apiDataSetMeta.ts +++ b/src/explore-education-statistics-common/src/services/types/apiDataSetMeta.ts @@ -3,6 +3,7 @@ import { GeographicLevelCode } from '@common/utils/locationLevelsMap'; export interface Filter { id: string; hint: string; + column: string; label: string; } @@ -20,6 +21,7 @@ export interface GeographicLevel { export interface IndicatorOption { id: string; label: string; + column: string; unit?: string; } diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/__tests__/DataSetFilePage.test.tsx b/src/explore-education-statistics-frontend/src/modules/data-catalogue/__tests__/DataSetFilePage.test.tsx index 070c5a59f16..0f52f455502 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/__tests__/DataSetFilePage.test.tsx +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/__tests__/DataSetFilePage.test.tsx @@ -308,7 +308,14 @@ describe('DataSetFilePage', () => { const testApiDataSetVersionChanges: ApiDataSetVersionChanges = { majorChanges: { filters: [ - { previousState: { id: 'filter-1', label: 'Filter 1', hint: '' } }, + { + previousState: { + id: 'filter-1', + column: 'filter_1', + label: 'Filter 1', + hint: '', + }, + }, ], }, minorChanges: {}, @@ -376,7 +383,14 @@ describe('DataSetFilePage', () => { const testApiDataSetVersionChanges: ApiDataSetVersionChanges = { majorChanges: { filters: [ - { previousState: { id: 'filter-1', label: 'Filter 1', hint: '' } }, + { + previousState: { + id: 'filter-1', + column: 'filter_1', + label: 'Filter 1', + hint: '', + }, + }, ], }, minorChanges: {}, diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/__tests__/DataSetFileApiChangelog.test.tsx b/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/__tests__/DataSetFileApiChangelog.test.tsx index 498fc2bced8..cdada94b656 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/__tests__/DataSetFileApiChangelog.test.tsx +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/components/__tests__/DataSetFileApiChangelog.test.tsx @@ -9,14 +9,24 @@ describe('DataSetFileApiChangelog', () => { majorChanges: { filters: [ { - previousState: { id: 'filter-1', label: 'Filter 1', hint: '' }, + previousState: { + id: 'filter-1', + column: 'filter_1', + label: 'Filter 1', + hint: '', + }, }, ], }, minorChanges: { filters: [ { - currentState: { id: 'filter-2', label: 'Filter 2', hint: '' }, + currentState: { + id: 'filter-2', + column: 'filter_2', + label: 'Filter 2', + hint: '', + }, }, ], }, @@ -41,13 +51,13 @@ describe('DataSetFileApiChangelog', () => { const majorChanges = within(screen.getByTestId('major-changes')); expect(majorChanges.getByTestId('deleted-filters')).toHaveTextContent( - 'Filter 1 (id: filter-1)', + 'Filter 1 (id: filter-1, column: filter_1)', ); const minorChanges = within(screen.getByTestId('minor-changes')); expect(minorChanges.getByTestId('added-filters')).toHaveTextContent( - 'Filter 2 (id: filter-2)', + 'Filter 2 (id: filter-2, column: filter_2)', ); }); @@ -58,7 +68,12 @@ describe('DataSetFileApiChangelog', () => { majorChanges: { filters: [ { - previousState: { id: 'filter-1', label: 'Filter 1', hint: '' }, + previousState: { + id: 'filter-1', + column: 'filter_1', + label: 'Filter 1', + hint: '', + }, }, ], }, @@ -72,7 +87,7 @@ describe('DataSetFileApiChangelog', () => { const majorChanges = within(screen.getByTestId('major-changes')); expect(majorChanges.getByTestId('deleted-filters')).toHaveTextContent( - 'Filter 1 (id: filter-1)', + 'Filter 1 (id: filter-1, column: filter_1)', ); expect(screen.queryByTestId('minor-changes')).not.toBeInTheDocument(); @@ -86,7 +101,12 @@ describe('DataSetFileApiChangelog', () => { minorChanges: { filters: [ { - currentState: { id: 'filter-2', label: 'Filter 2', hint: '' }, + currentState: { + id: 'filter-2', + column: 'filter_2', + label: 'Filter 2', + hint: '', + }, }, ], }, @@ -101,7 +121,7 @@ describe('DataSetFileApiChangelog', () => { const minorChanges = within(screen.getByTestId('minor-changes')); expect(minorChanges.getByTestId('added-filters')).toHaveTextContent( - 'Filter 2 (id: filter-2)', + 'Filter 2 (id: filter-2, column: filter_2)', ); }); @@ -113,7 +133,12 @@ describe('DataSetFileApiChangelog', () => { minorChanges: { filters: [ { - currentState: { id: 'filter-2', label: 'Filter 2', hint: '' }, + currentState: { + id: 'filter-2', + column: 'filter_2', + label: 'Filter 2', + hint: '', + }, }, ], }, From 6817fb355a3508c8e7e77bec90496a1bea851726 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Mon, 14 Oct 2024 14:47:09 +0100 Subject: [PATCH 67/80] EES-5572 Fix time periods not being naturally ordered for changelog --- .../Services/DataSetVersionChangeService.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionChangeService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionChangeService.cs index 54afe6f61b3..09157e4cc6c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionChangeService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Services/DataSetVersionChangeService.cs @@ -363,7 +363,7 @@ private async Task CreateIndicatorChanges( foreach (var (indicatorPublicId, newIndicator) in newMetas) { - if (!oldMetas.TryGetValue(indicatorPublicId, out var oldIndicator)) + if (!oldMetas.TryGetValue(indicatorPublicId, out _)) { // Indicator added metaAdditions.Add(new IndicatorMetaChange @@ -407,7 +407,7 @@ private async Task CreateTimePeriodChanges( var timePeriodMetaDeletions = oldTimePeriodMetas .Except(newTimePeriodMetas, TimePeriodMeta.CodePeriodComparer) - .OrderBy(timePeriodMeta => timePeriodMeta.Period) + .NaturalOrderBy(timePeriodMeta => timePeriodMeta.Period) .ThenBy(timePeriodMeta => timePeriodMeta.Code) .Select(timePeriodMeta => new TimePeriodMetaChange { @@ -419,7 +419,7 @@ private async Task CreateTimePeriodChanges( var timePeriodMetaAdditions = newTimePeriodMetas .Except(oldTimePeriodMetas, TimePeriodMeta.CodePeriodComparer) - .OrderBy(timePeriodMeta => timePeriodMeta.Period) + .NaturalOrderBy(timePeriodMeta => timePeriodMeta.Period) .ThenBy(timePeriodMeta => timePeriodMeta.Code) .Select(timePeriodMeta => new TimePeriodMetaChange { @@ -463,7 +463,7 @@ private async Task GetGeographicLevelMeta( .SingleAsync(m => m.DataSetVersionId == dataSetVersionId, cancellationToken); } - private async Task> GetIndicatorMetas( + private async Task> GetIndicatorMetas( Guid dataSetVersionId, CancellationToken cancellationToken) { From 35f7025cdefd0dcfe24ffe88c0a1793144d29cae Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Mon, 14 Oct 2024 14:55:15 +0100 Subject: [PATCH 68/80] EES-5572 Add ordering for geographic level changes --- .../Controllers/DataSetVersionsControllerTests.cs | 4 ++-- .../Services/DataSetVersionChangeService.cs | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetVersionsControllerTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetVersionsControllerTests.cs index 0d94d173b04..8e64f7b9d08 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetVersionsControllerTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Tests/Controllers/DataSetVersionsControllerTests.cs @@ -1141,12 +1141,12 @@ await TestApp.AddTestData(context => // Addition Assert.Null(minorChanges[0].PreviousState); Assert.NotNull(minorChanges[0].CurrentState); - Assert.Equal(GeographicLevel.Region, minorChanges[0].CurrentState!.Code); + Assert.Equal(GeographicLevel.LocalAuthority, minorChanges[0].CurrentState!.Code); // Addition Assert.Null(minorChanges[1].PreviousState); Assert.NotNull(minorChanges[1].CurrentState); - Assert.Equal(GeographicLevel.LocalAuthority, minorChanges[1].CurrentState!.Code); + Assert.Equal(GeographicLevel.Region, minorChanges[1].CurrentState!.Code); } [Fact] diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Services/DataSetVersionChangeService.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Services/DataSetVersionChangeService.cs index 86358ae7670..1f9986dc93d 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Services/DataSetVersionChangeService.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Api/Services/DataSetVersionChangeService.cs @@ -1,3 +1,4 @@ +using GovUk.Education.ExploreEducationStatistics.Common.Extensions; using GovUk.Education.ExploreEducationStatistics.Common.Model; using GovUk.Education.ExploreEducationStatistics.Common.Services.Interfaces.Security; using GovUk.Education.ExploreEducationStatistics.Public.Data.Api.Security.Extensions; @@ -194,12 +195,14 @@ private static DataSetVersionChangesViewModel MapChanges(DataSetVersion dataSetV [ ..currentLevels .Where(level => !previousLevels.Contains(level)) + .OrderBy(level => level.GetEnumLabel()) .Select(level => new GeographicLevelChangeViewModel { CurrentState = GeographicLevelViewModel.Create(level), }), ..previousLevels .Where(level => !currentLevels.Contains(level)) + .OrderBy(level => level.GetEnumLabel()) .Select(level => new GeographicLevelChangeViewModel { PreviousState = GeographicLevelViewModel.Create(level), From b4798dd31d4c7f9485267fad3bcb6342476602a3 Mon Sep 17 00:00:00 2001 From: Nicholas Tsim Date: Mon, 14 Oct 2024 15:03:38 +0100 Subject: [PATCH 69/80] EES-5572 Add missing service mock to `DataSetFilePage` tests --- .../modules/data-catalogue/__tests__/DataSetFilePage.test.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/explore-education-statistics-frontend/src/modules/data-catalogue/__tests__/DataSetFilePage.test.tsx b/src/explore-education-statistics-frontend/src/modules/data-catalogue/__tests__/DataSetFilePage.test.tsx index 0f52f455502..641b78f5208 100644 --- a/src/explore-education-statistics-frontend/src/modules/data-catalogue/__tests__/DataSetFilePage.test.tsx +++ b/src/explore-education-statistics-frontend/src/modules/data-catalogue/__tests__/DataSetFilePage.test.tsx @@ -10,6 +10,7 @@ import DataSetFilePage from '@frontend/modules/data-catalogue/DataSetFilePage'; import { screen, waitFor, within } from '@testing-library/react'; import React from 'react'; +jest.mock('@frontend/services/apiDataSetService'); jest.mock('@common/services/downloadService'); const downloadService = _downloadService as jest.Mocked< From 26e7bcecbeb8ef5d32ab7c4c6c4de7ebd90ee594 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Thu, 10 Oct 2024 14:24:17 +0100 Subject: [PATCH 70/80] EES-5560 - amending common keyword text selectors to accept either exact or fuzzy matching, ahead of needing this behaviour for testing the public changelog --- ...ge_approvals_as_publication_approver.robot | 4 +- .../bau/create_data_block_with_chart.robot | 7 +- .../bau/publish_content.robot | 3 +- .../admin_and_public/bau/publish_data.robot | 9 +- .../bau/publish_amend_and_cancel.robot | 2 +- .../bau/publish_release_and_amend.robot | 10 +- .../tests/general_public/fast_track.robot | 2 +- ...table_tool_absence_by_characteristic.robot | 4 +- .../robot-tests/tests/libs/admin-common.robot | 2 +- .../libs/admin/manage-content-common.robot | 14 +- tests/robot-tests/tests/libs/common.robot | 290 +++++++++++------- .../visual_testing/tables_and_charts.robot | 2 +- 12 files changed, 209 insertions(+), 140 deletions(-) diff --git a/tests/robot-tests/tests/admin/analyst/manage_approvals_as_publication_approver.robot b/tests/robot-tests/tests/admin/analyst/manage_approvals_as_publication_approver.robot index 524cf0f4772..4933c6feffc 100644 --- a/tests/robot-tests/tests/admin/analyst/manage_approvals_as_publication_approver.robot +++ b/tests/robot-tests/tests/admin/analyst/manage_approvals_as_publication_approver.robot @@ -67,7 +67,7 @@ Validate if Your approvals tab is correct Check that release link takes user to the correct release ${RELEASE_ROW}= get webelement testid:release-${RELEASE_NAME} - user clicks link by visible text Review this page ${RELEASE_ROW} + user clicks link containing text Review this page ${RELEASE_ROW} user waits until h1 is visible ${PUBLICATION_NAME} %{WAIT_MEDIUM} user waits until page contains title caption Edit release for Academic year 2026/27 @@ -80,7 +80,7 @@ Check that Your approvals tab methodology link takes user to the correct methodo user waits until h2 is visible Your approvals ${METHODOLOGY_ROW}= get webelement testid:methodology-${PUBLICATION_NAME} - ${PUBLICATION_NAME} - user clicks link by visible text Review this page ${METHODOLOGY_ROW} + user clicks link containing text Review this page ${METHODOLOGY_ROW} user waits until h1 is visible ${PUBLICATION_NAME} user waits until page contains title caption Edit methodology diff --git a/tests/robot-tests/tests/admin/bau/create_data_block_with_chart.robot b/tests/robot-tests/tests/admin/bau/create_data_block_with_chart.robot index 05bd8417b5f..0314c3021d5 100644 --- a/tests/robot-tests/tests/admin/bau/create_data_block_with_chart.robot +++ b/tests/robot-tests/tests/admin/bau/create_data_block_with_chart.robot @@ -29,7 +29,8 @@ Create test publication and release via API Upload subject user navigates to draft release page from dashboard ${PUBLICATION_NAME} ... Academic year 2025/26 - user uploads subject and waits until complete UI test subject upload-file-test.csv upload-file-test.meta.csv + user uploads subject and waits until complete UI test subject upload-file-test.csv + ... upload-file-test.meta.csv Navigate to 'Footnotes' page user waits until page finishes loading @@ -207,7 +208,6 @@ Validate data block is in list user checks table column heading contains 1 4 Created date testid:dataBlocks user checks table column heading contains 1 5 Actions testid:dataBlocks - user checks table body has x rows 1 testid:dataBlocks user checks table cell contains 1 1 ${DATABLOCK_NAME} testid:dataBlocks user checks table cell contains 1 3 No testid:dataBlocks @@ -488,7 +488,7 @@ Add reference line user clicks button Add new line user chooses select option id:chartAxisConfiguration-major-referenceLines-position 2005 user enters text into element id:chartAxisConfiguration-major-referenceLines-label Reference line 1 - user clicks button Add + user clicks button Add exact_match=${TRUE} Validate basic line chart preview user waits until element contains line chart id:chartBuilderPreview @@ -552,7 +552,6 @@ Save chart and validate marked as 'Has chart' in data blocks list user checks table column heading contains 1 1 Name testid:dataBlocks user checks table column heading contains 1 2 Has chart testid:dataBlocks - user checks table body has x rows 1 user checks table cell contains 1 1 ${DATABLOCK_NAME} testid:dataBlocks user checks table cell contains 1 2 Yes testid:dataBlocks diff --git a/tests/robot-tests/tests/admin_and_public/bau/publish_content.robot b/tests/robot-tests/tests/admin_and_public/bau/publish_content.robot index 770b1914695..351cd7fd3ab 100644 --- a/tests/robot-tests/tests/admin_and_public/bau/publish_content.robot +++ b/tests/robot-tests/tests/admin_and_public/bau/publish_content.robot @@ -48,7 +48,8 @@ Add text block with link to absence glossary entry to accordion section ${block}= user starts editing accordion section text block Test section 1 ... ${RELEASE_CONTENT_EDITABLE_ACCORDION} ${toolbar}= get editor toolbar ${block} - ${insert}= user gets button element Insert ${toolbar} + ${insert}= get child element parent_locator=${toolbar} + ... child_locator=xpath://button[@data-cke-tooltip-text="Insert"] user clicks element ${insert} ${button}= user gets button element Insert glossary link ${toolbar} user clicks element ${button} diff --git a/tests/robot-tests/tests/admin_and_public/bau/publish_data.robot b/tests/robot-tests/tests/admin_and_public/bau/publish_data.robot index 681e3713fd7..57f15f5e4e8 100644 --- a/tests/robot-tests/tests/admin_and_public/bau/publish_data.robot +++ b/tests/robot-tests/tests/admin_and_public/bau/publish_data.robot @@ -72,8 +72,10 @@ Verify new release summary user checks summary list contains Release type Accredited official statistics Upload subjects to release - user uploads subject and waits until complete ${SUBJECT_1_NAME} tiny-two-filters.csv tiny-two-filters.meta.csv - user uploads subject and waits until complete ${SUBJECT_2_NAME} upload-file-test.csv upload-file-test-with-filter.meta.csv + user uploads subject and waits until complete ${SUBJECT_1_NAME} tiny-two-filters.csv + ... tiny-two-filters.meta.csv + user uploads subject and waits until complete ${SUBJECT_2_NAME} upload-file-test.csv + ... upload-file-test-with-filter.meta.csv Navigate to Footnotes page user clicks link Footnotes @@ -502,7 +504,8 @@ Add text block with link to a featured table to accordion section ${block}= user starts editing accordion section text block Test section 1 ... ${RELEASE_CONTENT_EDITABLE_ACCORDION} ${toolbar}= get editor toolbar ${block} - ${insert}= user gets button element Insert ${toolbar} + ${insert}= get child element parent_locator=${toolbar} + ... child_locator=xpath://button[@data-cke-tooltip-text="Insert"] user clicks element ${insert} ${button}= user gets button element Insert featured table link ${toolbar} user clicks element ${button} diff --git a/tests/robot-tests/tests/admin_and_public_2/bau/publish_amend_and_cancel.robot b/tests/robot-tests/tests/admin_and_public_2/bau/publish_amend_and_cancel.robot index 9583f4db208..f3bda5ff9ad 100644 --- a/tests/robot-tests/tests/admin_and_public_2/bau/publish_amend_and_cancel.robot +++ b/tests/robot-tests/tests/admin_and_public_2/bau/publish_amend_and_cancel.robot @@ -451,7 +451,7 @@ Verify that the Dates data block accordion is unchanged user checks chart title contains ${section} Dates table title user checks infographic chart contains alt ${section} Sample alt text - user clicks link by visible text Table ${section} + user clicks link containing text Table ${section} user waits until parent contains element ${section} ... xpath:.//*[@data-testid="dataTableCaption" and text()="Dates table title"] user waits until parent contains element ${section} xpath:.//*[.="Source: Dates source"] diff --git a/tests/robot-tests/tests/admin_and_public_2/bau/publish_release_and_amend.robot b/tests/robot-tests/tests/admin_and_public_2/bau/publish_release_and_amend.robot index d933eda5c03..a8f0c7615d2 100644 --- a/tests/robot-tests/tests/admin_and_public_2/bau/publish_release_and_amend.robot +++ b/tests/robot-tests/tests/admin_and_public_2/bau/publish_release_and_amend.robot @@ -232,7 +232,7 @@ Verify data block is updated correctly ${datablock}= set variable testid:Data block - ${DATABLOCK_NAME} user checks chart title contains ${datablock} Updated dates table title - user clicks link by visible text Table ${datablock} + user clicks link containing text Table ${datablock} user waits until parent contains element ${datablock} ... xpath:.//*[@data-testid="dataTableCaption" and text()="Updated dates table title"] user waits until parent contains element ${datablock} xpath:.//*[.="Source: Updated dates source"] @@ -407,7 +407,7 @@ Verify Dates data block accordion section user checks chart title contains ${section} Updated dates table title user checks infographic chart contains alt ${section} Sample alt text - user clicks link by visible text Table ${section} + user clicks link containing text Table ${section} user waits until parent contains element ${section} ... xpath:.//*[@data-testid="dataTableCaption" and text()="Updated dates table title"] user waits until parent contains element ${section} xpath:.//*[.="Source: Updated dates source"] @@ -434,7 +434,7 @@ Verify Dates data block table has footnotes Verify Dates data block Fast Track page ${release_url}= user gets url - user clicks link by visible text Explore data testid:Data block - Dates data block name-table-tab + user clicks link containing text Explore data testid:Data block - Dates data block name-table-tab user waits until page contains title Create your own tables user waits until page contains This is the latest data @@ -882,7 +882,7 @@ Verify amendment Dates data block accordion section user checks chart title contains ${section} Amended sample title user checks infographic chart contains alt ${section} Amended sample alt text - user clicks link by visible text Table ${section} + user clicks link containing text Table ${section} user waits until parent contains element ${section} ... xpath:.//*[@data-testid="dataTableCaption" and text()="Amended dates table title"] user waits until parent contains element ${section} xpath:.//*[.="Source: Amended dates source"] @@ -915,7 +915,7 @@ Verify amendment Dates data block table has footnotes Verify amendment Dates data block Fast Track page ${release_url}= user gets url - user clicks link by visible text Explore data testid:Data block - Dates data block name-table-tab + user clicks link containing text Explore data testid:Data block - Dates data block name-table-tab user waits until page contains title Create your own tables user waits until page contains This is the latest data diff --git a/tests/robot-tests/tests/general_public/fast_track.robot b/tests/robot-tests/tests/general_public/fast_track.robot index 27f012c57a0..a0d96c51049 100644 --- a/tests/robot-tests/tests/general_public/fast_track.robot +++ b/tests/robot-tests/tests/general_public/fast_track.robot @@ -20,7 +20,7 @@ Click fast track link for 'Pupil absence rates' data block user scrolls to accordion section Pupil absence rates id:content user scrolls to element testid:Data block - Generic data block - National user waits until h3 is visible Explore and edit this data online - user clicks link by visible text Explore data testid:Data block - Generic data block - National + user clicks link containing text Explore data testid:Data block - Generic data block - National Validate Publication selected step option user waits until h1 is visible Create your own tables %{WAIT_SMALL} diff --git a/tests/robot-tests/tests/general_public/table_tool_absence_by_characteristic.robot b/tests/robot-tests/tests/general_public/table_tool_absence_by_characteristic.robot index 0db750d00e2..14792ac8799 100644 --- a/tests/robot-tests/tests/general_public/table_tool_absence_by_characteristic.robot +++ b/tests/robot-tests/tests/general_public/table_tool_absence_by_characteristic.robot @@ -137,8 +137,8 @@ Reorder Gender to be column group user clicks button Move and reorder table headers # Column group needs to be inside the viewport user scrolls to element xpath://button[text()="Update and view reordered table"] - user clicks button Move testId:rowGroups-0 - user clicks button Move testId:rowGroups-0 + user clicks button Move testId:rowGroups-0 exact_match=${TRUE} + user clicks button Move Characteristic to columns testId:rowGroups-0 user clicks button Done testId:columnGroups-1 Move Gender to be first column group diff --git a/tests/robot-tests/tests/libs/admin-common.robot b/tests/robot-tests/tests/libs/admin-common.robot index b459ac6a390..45a30572429 100644 --- a/tests/robot-tests/tests/libs/admin-common.robot +++ b/tests/robot-tests/tests/libs/admin-common.robot @@ -119,7 +119,7 @@ user navigates to release page from dashboard ${ROW}= user gets table row ${RELEASE_NAME} testid:${RELEASE_TABLE_TESTID} user scrolls to element ${ROW} - user clicks link by visible text ${LINK_TEXT} ${ROW} + user clicks link containing text ${LINK_TEXT} ${ROW} user waits until h2 is visible Release summary %{WAIT_SMALL} user navigates to draft release page from dashboard diff --git a/tests/robot-tests/tests/libs/admin/manage-content-common.robot b/tests/robot-tests/tests/libs/admin/manage-content-common.robot index 92c9ae400b3..3a2ea2d4546 100644 --- a/tests/robot-tests/tests/libs/admin/manage-content-common.robot +++ b/tests/robot-tests/tests/libs/admin/manage-content-common.robot @@ -186,9 +186,9 @@ user chooses and embeds data block [Arguments] ... ${datablock_name} user chooses select option css:select[name="selectedDataBlock"] ${datablock_name} - user waits until button is enabled Embed %{WAIT_SMALL} - user clicks button Embed - user waits until page does not contain button Embed %{WAIT_MEDIUM} + user waits until button is enabled Embed %{WAIT_SMALL} exact_match=${TRUE} + user clicks button Embed exact_match=${TRUE} + user waits until page does not contain button Embed %{WAIT_MEDIUM} exact_match=${TRUE} user waits until page finishes loading user opens nth editable accordion section @@ -468,12 +468,14 @@ user adds image to accordion section text block with retry ... ${FILES_DIR}${filename} user scrolls up 300 - wait until keyword succeeds ${timeout} %{WAIT_SMALL} sec user clicks button Change image text alternative + wait until keyword succeeds ${timeout} %{WAIT_SMALL} sec user clicks button + ... Change image text alternative user enters text into element label:Text alternative ${alt_text} user clicks element css:button.ck-button-save sleep 5 user scrolls up 100 - wait until keyword succeeds ${timeout} %{WAIT_SMALL} sec user clicks element xpath://div[@title="Insert paragraph after block"] + wait until keyword succeeds ${timeout} %{WAIT_SMALL} sec user clicks element + ... xpath://div[@title="Insert paragraph after block"] # wait for the API to save the image and for the src attribute to be updated before continuing user waits until parent contains element ${block} @@ -566,7 +568,7 @@ user adds link to accordion section text block ${button}= user gets button element Link ${toolbar} user clicks element ${button} user enters text into element label:Link URL ${url} - + # Save user presses keys TAB user presses keys ENTER diff --git a/tests/robot-tests/tests/libs/common.robot b/tests/robot-tests/tests/libs/common.robot index 5b9cf9d70f3..d024fcd14a3 100644 --- a/tests/robot-tests/tests/libs/common.robot +++ b/tests/robot-tests/tests/libs/common.robot @@ -12,15 +12,15 @@ Resource ./table_tool.robot *** Variables *** -${browser}= chrome -${headless}= 1 -${FILES_DIR}= ${EXECDIR}${/}tests${/}files${/} -${PUBLIC_API_FILES_DIR}= ${EXECDIR}${/}tests${/}files${/}public-api-data-files${/} -${UNZIPPED_FILES_DIR}= ${EXECDIR}${/}tests${/}files${/}.unzipped-seed-data-files${/} -${DOWNLOADS_DIR}= ${EXECDIR}${/}test-results${/}downloads${/} -${timeout}= %{TIMEOUT} -${implicit_wait}= %{IMPLICIT_WAIT} -${prompt_to_continue_on_failure}= 0 +${browser} chrome +${headless} 1 +${FILES_DIR} ${EXECDIR}${/}tests${/}files${/} +${PUBLIC_API_FILES_DIR} ${EXECDIR}${/}tests${/}files${/}public-api-data-files${/} +${UNZIPPED_FILES_DIR} ${EXECDIR}${/}tests${/}files${/}.unzipped-seed-data-files${/} +${DOWNLOADS_DIR} ${EXECDIR}${/}test-results${/}downloads${/} +${timeout} %{TIMEOUT} +${implicit_wait} %{IMPLICIT_WAIT} +${prompt_to_continue_on_failure} 0 *** Keywords *** @@ -220,50 +220,59 @@ user waits until element contains testid user waits until parent contains element ${element} css:[data-testid="${testid}"] timeout=${wait} user waits until page contains accordion section - [Arguments] ${section_title} ${wait}=${timeout} + [Arguments] ${section_title} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${section_title} ${exact_match} user waits until page contains element - ... xpath://button[@class='govuk-accordion__section-button'][.//span[text()="${section_title}"]] ${wait} + ... xpath://button[@class='govuk-accordion__section-button'][.//span[${text_matcher}]] ${wait} user waits until page does not contain accordion section - [Arguments] ${section_title} ${wait}=${timeout} + [Arguments] ${section_title} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${section_title} ${exact_match} user waits until page does not contain element - ... xpath://button[@class='govuk-accordion__section-button'][.//span[text()="${section_title}"]] ${wait} + ... xpath://button[@class='govuk-accordion__section-button'][.//span[${text_matcher}]] ${wait} user verifies accordion is open - [Arguments] ${section_text} + [Arguments] ${section_text} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${section_text} ${exact_match} user waits until page contains element - ... xpath://button[@class='govuk-accordion__section-button'][.//span[text()="${section_text}"] and @aria-expanded="true"] + ... xpath://button[@class='govuk-accordion__section-button'][.//span[${text_matcher}] and @aria-expanded="true"] user verifies accordion is closed - [Arguments] ${section_text} + [Arguments] ${section_text} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${section_text} ${exact_match} user waits until page contains element - ... xpath://button[@class='govuk-accordion__section-button'][.//span[text()="${section_text}"] and @aria-expanded="false"] + ... xpath://button[@class='govuk-accordion__section-button'][.//span[${text_matcher}] and @aria-expanded="false"] user checks there are x accordion sections [Arguments] ${count} ${parent}=css:body user waits until parent contains element ${parent} css:[data-testid="accordionSection"] count=${count} user checks accordion is in position - [Arguments] ${section_text} ${position} ${parent}=css:[data-testid="accordion"] + [Arguments] ${section_text} ${position} ${parent}=css:[data-testid="accordion"] ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${section_text} ${exact_match} user waits until parent contains element ${parent} - ... xpath:(.//*[@data-testid="accordionSection"])[${position}]//span[starts-with(text(), "${section_text}")] + ... xpath:(.//*[@data-testid="accordionSection"])[${position}]//span[${text_matcher}] user waits until accordion section contains text - [Arguments] ${section_text} ${text} ${wait}=${timeout} + [Arguments] ${section_text} ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} ${section}= user gets accordion section content element ${section_text} - user waits until parent contains element ${section} xpath:.//*[text()="${text}"] timeout=${wait} + user waits until parent contains element ${section} xpath:.//*[${text_matcher}] timeout=${wait} user gets accordion header button element - [Arguments] ${heading_text} ${parent}=css:[data-testid="accordion"] - ${button}= get child element ${parent} xpath:.//button[@aria-expanded and contains(., "${heading_text}")] + [Arguments] ${heading_text} ${parent}=css:[data-testid="accordion"] ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${heading_text} ${exact_match} + ${button}= get child element ${parent} xpath:.//button[@aria-expanded and ${text_matcher}] [Return] ${button} user opens accordion section [Arguments] ... ${heading_text} ... ${parent}=css:[data-testid="accordion"] + ... ${exact_match}=${FALSE} ${header_button}= user gets accordion header button element ${heading_text} ${parent} + ... exact_match=${exact_match} ${accordion}= user opens accordion section with accordion header ${header_button} ${parent} [Return] ${accordion} @@ -290,22 +299,25 @@ user opens accordion section with accordion header [Return] ${accordion} user closes accordion section - [Arguments] ${heading_text} ${parent}=css:[data-testid="accordion"] + [Arguments] ${heading_text} ${parent}=css:[data-testid="accordion"] ${exact_match}=${FALSE} ${header_button}= user gets accordion header button element ${heading_text} ${parent} - user closes accordion section with accordion header ${header_button} ${parent} + ... exact_match=${exact_match} + user closes accordion section with accordion header ${header_button} ${parent} exact_match=${exact_match} user closes accordion section with id [Arguments] ... ${id} ... ${parent}=css:[data-testid="accordion"] + ... ${exact_match}=${FALSE} ${header_button}= get child element ${parent} id:${id}-heading - user closes accordion section with accordion header ${header_button} ${parent} + user closes accordion section with accordion header ${header_button} ${parent} exact_match=${exact_match} user closes accordion section with accordion header [Arguments] ... ${header_button} ... ${parent}=css:[data-testid="accordion"] + ... ${exact_match}=${FALSE} ${is_expanded}= get element attribute ${header_button} aria-expanded IF '${is_expanded}' != 'false' @@ -314,8 +326,9 @@ user closes accordion section with accordion header user checks element attribute value should be ${header_button} aria-expanded false user gets accordion section content element - [Arguments] ${heading_text} ${parent}=css:[data-testid="accordion"] + [Arguments] ${heading_text} ${parent}=css:[data-testid="accordion"] ${exact_match}=${FALSE} ${header_button}= user gets accordion header button element ${heading_text} ${parent} + ... exact_match=${exact_match} ${content_id}= get element attribute ${header_button} aria-controls ${content}= get child element ${parent} css:[id="${content_id}"] [Return] ${content} @@ -327,9 +340,11 @@ user gets accordion section content element from heading element [Return] ${content} user scrolls to accordion section - [Arguments] ${heading_text} ${parent}=css:[data-testid="accordion"] + [Arguments] ${heading_text} ${parent}=css:[data-testid="accordion"] ${exact_match}=${FALSE} ${header_button}= user gets accordion header button element ${heading_text} ${parent} + ... exact_match=${exact_match} ${content}= user gets accordion section content element ${heading_text} ${parent} + ... exact_match=${exact_match} user scrolls to element ${header_button} # Workaround to get lazy loaded data blocks to render user scrolls down 1 @@ -344,7 +359,7 @@ user checks page does not contain testid user checks testid element contains [Arguments] ${id} ${text} - user waits until element contains css:[data-testid="${id}"] ${text} + user waits until element contains testid:${id} ${text} user gets testid element [Arguments] ${id} ${wait}=${timeout} ${parent}=css:body @@ -361,6 +376,8 @@ user checks element contains child element ... ${element} ... ${child_element} user waits until parent contains element ${element} ${child_element} + ${child}= get child element ${element} ${child_element} + RETURN ${child} user checks element does not contain child element [Arguments] @@ -369,8 +386,9 @@ user checks element does not contain child element user waits until parent does not contain element ${element} ${child_element} user checks element contains - [Arguments] ${element} ${text} - user waits until parent contains element ${element} xpath://*[contains(.,"${text}")] + [Arguments] ${element} ${text} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user waits until parent contains element ${element} xpath://*[${text_matcher}] user checks element contains button [Arguments] @@ -491,22 +509,20 @@ user clicks link by index ${button}= get webelement ${xpath} user clicks element ${button} ${parent} -user clicks link by visible text - [Arguments] ${text} ${parent}=css:body - user clicks element xpath:.//a[text()="${text}"] ${parent} - user clicks link containing text - [Arguments] ${text} ${parent}=css:body - user clicks element xpath:.//a[contains(text(), "${text}")] ${parent} + [Arguments] ${text} ${parent}=css:body ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user clicks element xpath:.//a[${text_matcher}] ${parent} user clicks button - [Arguments] ${text} ${parent}=css:body - ${button}= user gets button element ${text} ${parent} + [Arguments] ${text} ${parent}=css:body ${exact_match}=${FALSE} + ${button}= user gets button element ${text} ${parent} exact_match=${exact_match} user clicks element ${button} user clicks button by index - [Arguments] ${text} ${index}=1 ${parent}=css:body - ${xpath}= set variable (//button[text()='${text}'])[${index}] + [Arguments] ${text} ${index}=1 ${parent}=css:body ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + ${xpath}= set variable (//button[${text_matcher}])[${index}] ${button}= get webelement ${xpath} user clicks element ${button} ${parent} @@ -518,91 +534,124 @@ user waits until button is clickable element should be enabled xpath=//button[text()="${button_text}"] user clicks button containing text - [Arguments] ${text} ${parent}=css:body - user clicks element xpath://button[contains(text(), "${text}")] ${parent} + [Arguments] ${text} ${parent}=css:body ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user clicks element xpath://button[${text_matcher}] ${parent} user waits until page contains button - [Arguments] ${text} ${wait}=${timeout} - user waits until page contains element xpath://button[text()="${text}" or .//*[text()="${text}"]] ${wait} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user waits until page contains element xpath://button[${text_matcher}] + ... ${wait} user checks page contains button - [Arguments] ${text} - user checks page contains element xpath://button[text()="${text}" or .//*[text()="${text}"]] + [Arguments] ${text} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user checks page contains element xpath://button[${text_matcher}] user checks page does not contain button - [Arguments] ${text} - user checks page does not contain element xpath://button[text()="${text}" or .//*[text()="${text}"]] + [Arguments] ${text} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user checks page does not contain element xpath://button[${text_matcher}] user waits until page does not contain button - [Arguments] ${text} ${wait}=${timeout} - user waits until page does not contain element xpath://button[text()="${text}" or .//*[text()="${text}"]] - ... ${wait} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user waits until page does not contain element + ... xpath://button[${text_matcher}] ${wait} user waits until button is enabled - [Arguments] ${text} ${wait}=${timeout} - user waits until element is enabled xpath://button[text()="${text}" or .//*[text()="${text}"]] ${wait} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user waits until element is enabled xpath://button[${text_matcher}] + ... ${wait} user waits until parent contains button - [Arguments] ${parent} ${text} ${wait}=${timeout} + [Arguments] ${parent} ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until parent contains element ${parent} - ... xpath:.//button[text()="${text}" or .//*[text()="${text}"]] ${wait} + ... xpath://button[${text_matcher}] ${wait} user waits until parent does not contain button - [Arguments] ${parent} ${text} ${wait}=${timeout} + [Arguments] ${parent} ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until parent does not contain element ${parent} - ... xpath:.//button[text()="${text}" or .//*[text()="${text}"]] ${wait} + ... xpath://button[${text_matcher}] ${wait} user waits until parent does not contain - [Arguments] ${parent} ${text} ${wait}=${timeout} + [Arguments] ${parent} ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until parent does not contain element ${parent} - ... .//*[contains(text(),"${text}")] ${wait} + ... //button[${text_matcher}] ${wait} user gets button element - [Arguments] ${text} ${parent}=css:body + [Arguments] ${text} ${parent}=css:body ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until parent contains button ${parent} ${text} - ${button}= get child element ${parent} xpath:.//button[text()="${text}" or .//*[text()="${text}"]] + ${button}= get child element ${parent} xpath://button[${text_matcher}] [Return] ${button} +get xpath text matcher + [Arguments] ${text} ${exact_match}=${FALSE} + IF "${exact_match}" == "${TRUE}" + ${expression}= Set Variable text()="${text}" + ELSE + ${expression}= Set Variable contains(., "${text}") + END + RETURN ${expression} + user checks page contains tag - [Arguments] ${text} - user checks page contains element xpath://*[contains(@class, "govuk-tag")][text()="${text}"] + [Arguments] ${text} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user checks page contains element xpath://*[contains(@class, "govuk-tag")][${text_matcher}] user waits until h1 is visible - [Arguments] ${text} ${wait}=${timeout} - user waits until element is visible xpath://h1[text()="${text}"] ${wait} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user waits until element is visible xpath://h1[${text_matcher}] ${wait} user waits until h1 is not visible - [Arguments] ${text} ${wait}=%{WAIT_SMALL} - user waits until element is not visible xpath://h1[text()="${text}"] ${wait} + [Arguments] ${text} ${wait}=%{WAIT_SMALL} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user waits until element is not visible xpath://h1[${text_matcher}] ${wait} user waits until h2 is visible - [Arguments] ${text} ${wait}=${timeout} - user waits until element is visible xpath://h2[text()="${text}"] ${wait} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user waits until element is visible xpath://h2[${text_matcher}] ${wait} user waits until h2 is not visible - [Arguments] ${text} ${wait}=${timeout} - user waits until element is not visible xpath://h2[text()="${text}"] ${wait} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user waits until element is not visible xpath://h2[${text_matcher}] ${wait} user waits until h3 is visible - [Arguments] ${text} ${wait}=${timeout} - user waits until element is visible xpath://h3[text()="${text}"] ${wait} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user waits until element is visible xpath://h3[${text_matcher}] ${wait} user waits until h3 is not visible - [Arguments] ${text} ${wait}=${timeout} - user waits until element is not visible xpath://h3[text()="${text}"] ${wait} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user waits until element is not visible xpath://h3[${text_matcher}] ${wait} user waits until legend is visible - [Arguments] ${text} ${wait}=${timeout} - user waits until element is visible xpath://legend[text()="${text}"] ${wait} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user waits until element is visible xpath://legend[${text_matcher}] ${wait} user waits until page contains title - [Arguments] ${text} ${wait}=${timeout} - user waits until page contains element xpath://h1[@data-testid="page-title" and text()="${text}"] ${wait} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user waits until page contains element xpath://h1[@data-testid="page-title" and ${text_matcher}] + ... ${wait} user waits until page contains title caption - [Arguments] ${text} ${wait}=${timeout} - user waits until page contains element xpath://span[@data-testid="page-title-caption" and text()="${text}"] - ... ${wait} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user waits until page contains element + ... xpath://span[@data-testid="page-title-caption" and ${text_matcher}] ${wait} user selects newly opened window switch window locator=NEW @@ -715,23 +764,28 @@ user checks page contains link [Arguments] ... ${text} ... ${parent}=css:body + ... ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until parent contains element ${parent} - ... xpath:.//a[contains(text(), "${text}")] + ... xpath:.//a[${text_matcher}] user checks page contains link with text and url [Arguments] ... ${text} ... ${href} ... ${parent}=css:body + ... ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until parent contains element ${parent} - ... xpath:.//a[@href="${href}" and contains(text(), "${text}")] + ... xpath:.//a[@href="${href}" and ${text_matcher}] user opens details dropdown - [Arguments] ${text} ${parent}=css:body + [Arguments] ${text} ${parent}=css:body ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until parent contains element ${parent} - ... xpath:.//details/summary[contains(., "${text}") and @aria-expanded] %{WAIT_SMALL} - ${details}= get child element ${parent} xpath:.//details[summary[contains(., "${text}")]] - ${summary}= get child element ${parent} xpath:.//details/summary[contains(., "${text}")] + ... xpath:.//details/summary[${text_matcher} and @aria-expanded] %{WAIT_SMALL} + ${details}= get child element ${parent} xpath:.//details[summary[${text_matcher}]] + ${summary}= get child element ${parent} xpath:.//details/summary[${text_matcher}] user waits until element is visible ${summary} %{WAIT_SMALL} ${is_expanded}= get element attribute ${summary} aria-expanded IF '${is_expanded}' != 'true' @@ -741,10 +795,11 @@ user opens details dropdown [Return] ${details} user closes details dropdown - [Arguments] ${text} ${parent}=css:body + [Arguments] ${text} ${parent}=css:body ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until parent contains element ${parent} ... xpath:.//details/summary[contains(., "${text}") and @aria-expanded] - ${summary}= get child element ${parent} xpath:.//details/summary[contains(., "${text}")] + ${summary}= get child element ${parent} xpath:.//details/summary[${text_matcher}] user waits until element is visible ${summary} ${is_expanded}= get element attribute ${summary} aria-expanded IF '${is_expanded}' != 'false' @@ -753,25 +808,29 @@ user closes details dropdown user checks element attribute value should be ${summary} aria-expanded false user gets details content element - [Arguments] ${text} ${parent}=css:body ${wait}=${timeout} - user waits until parent contains element ${parent} xpath:.//details/summary[contains(., "${text}")] + [Arguments] ${text} ${parent}=css:body ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user waits until parent contains element ${parent} xpath:.//details/summary[${text_matcher}] ... timeout=${wait} - ${summary}= get child element ${parent} xpath:.//details/summary[contains(., "${text}")] + ${summary}= get child element ${parent} xpath:.//details/summary[${text_matcher}] ${content_id}= get element attribute ${summary} aria-controls ${content}= get child element ${parent} id:${content_id} [Return] ${content} user waits until page contains details dropdown - [Arguments] ${text} ${wait}=${timeout} - user waits until page contains element xpath:.//details/summary[contains(., "${text}")] ${wait} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user waits until page contains element xpath:.//details/summary[${text_matcher}] ${wait} user checks page for details dropdown - [Arguments] ${text} - user checks page contains element xpath:.//details/summary[contains(., "${text}")] + [Arguments] ${text} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user checks page contains element xpath:.//details/summary[${text_matcher}] user scrolls to details dropdown - [Arguments] ${text} ${wait}=${timeout} - user scrolls to element xpath:.//details/summary[contains(., "${text}")] + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user scrolls to element xpath:.//details/summary[${text_matcher}] user checks publication bullet contains link [Arguments] ${publication} ${link} @@ -821,14 +880,16 @@ user checks radio is checked user checks page contains element xpath://label[text()="${label}"]/../input[@type="radio" and @checked] user checks radio in position has label - [Arguments] ${position} ${label} + [Arguments] ${position} ${label} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${label} ${exact_match} user checks page contains element - ... xpath://*[contains(@data-testid, "Radio item for ")][${position}]//label[contains(text(), "${label}")] + ... xpath://*[contains(@data-testid, "Radio item for ")][${position}]//label[${text_matcher}] user clicks checkbox - [Arguments] ${label} - user scrolls to element xpath://label[text()="${label}" or strong[text()="${label}"]]/../input[@type="checkbox"] - user clicks element xpath://label[text()="${label}" or strong[text()="${label}"]]/../input[@type="checkbox"] + [Arguments] ${label} ${exact_match}=${TRUE} + ${text_matcher}= get xpath text matcher ${label} ${exact_match} + user scrolls to element xpath://label[${text_matcher} or strong[${text_matcher}]]/../input[@type="checkbox"] + user clicks element xpath://label[${text_matcher} or strong[${text_matcher}]]/../input[@type="checkbox"] user clicks checkbox by selector [Arguments] ${locator} @@ -836,14 +897,16 @@ user clicks checkbox by selector user clicks element ${locator} user checks checkbox is checked - [Arguments] ${label} + [Arguments] ${label} ${exact_match}=${TRUE} + ${text_matcher}= get xpath text matcher ${label} ${exact_match} user checks checkbox input is checked - ... xpath://label[text()="${label}" or strong[text()="${label}"]]/../input[@type="checkbox"] + ... xpath://label[${text_matcher} or strong[${text_matcher}]]/../input[@type="checkbox"] user checks checkbox is not checked - [Arguments] ${label} + [Arguments] ${label} ${exact_match}=${TRUE} + ${text_matcher}= get xpath text matcher ${label} ${exact_match} user checks checkbox input is not checked - ... xpath://label[text()="${label}" or strong[text()="${label}"]]/../input[@type="checkbox"] + ... xpath://label[${text_matcher} or strong[${text_matcher}]]/../input[@type="checkbox"] user checks checkbox input is checked [Arguments] ${selector} @@ -856,9 +919,10 @@ user checks checkbox input is not checked checkbox should not be selected ${selector} user checks checkbox in position has label - [Arguments] ${position} ${label} + [Arguments] ${position} ${label} ${exact_match}=${TRUE} + ${text_matcher}= get xpath text matcher ${label} ${exact_match} user checks page contains element - ... xpath://*[contains(@data-testid,"Checkbox item for ")][${position}]//label[contains(text(), "${label}")] + ... xpath://*[contains(@data-testid,"Checkbox item for ")][${position}]//label[${text_matcher}] user checks list has x items [Arguments] ${locator} ${count} ${parent}=css:body @@ -968,7 +1032,7 @@ user gets data block from parent user gets data block table from parent [Arguments] ${data_block_name} ${parent} ${data_block}= user gets data block from parent ${data_block_name} ${parent} - user clicks link by visible text Table ${data_block} + user clicks link containing text Table ${data_block} ${data_block_id}= get element attribute ${data_block} id ${data_block_table}= get child element ${data_block} id:${data_block_id}-tables [Return] ${data_block_table} @@ -976,7 +1040,7 @@ user gets data block table from parent user gets data block chart from parent [Arguments] ${data_block_name} ${parent} ${data_block}= user gets data block from parent ${data_block_name} ${parent} - user clicks link by visible text Chart ${data_block} + user clicks link containing text Chart ${data_block} ${data_block_id}= get element attribute ${data_block} id ${data_block_chart}= get child element ${data_block} id:${data_block_id}-chart [Return] ${data_block_chart} diff --git a/tests/robot-tests/tests/visual_testing/tables_and_charts.robot b/tests/robot-tests/tests/visual_testing/tables_and_charts.robot index e91697dc647..8b90ae64637 100644 --- a/tests/robot-tests/tests/visual_testing/tables_and_charts.robot +++ b/tests/robot-tests/tests/visual_testing/tables_and_charts.robot @@ -92,7 +92,7 @@ Check Content Block Table ${tables_tab}= set variable dataBlock-${content_block.content_block_id}-tables user waits until parent contains element ${data_block} id:${tables_tab} user waits until element is enabled id:${tables_tab} - user clicks link by visible text Table ${data_block} + user clicks link containing text Table ${data_block} ELSE ${tables_tab}= set variable dataBlock-${content_block.content_block_id} user scrolls to element ${data_block} From 1d35ed4b495c234eae2f963c5a693451a454a042 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Thu, 10 Oct 2024 14:25:09 +0100 Subject: [PATCH 71/80] EES-5560 - adding Robot test for public changelog --- .../public_api_resolve_mapping_statuses.robot | 129 +++++++++++------- 1 file changed, 76 insertions(+), 53 deletions(-) diff --git a/tests/robot-tests/tests/public_api/public_api_resolve_mapping_statuses.robot b/tests/robot-tests/tests/public_api/public_api_resolve_mapping_statuses.robot index a8b6ac1cb04..1d699564b86 100644 --- a/tests/robot-tests/tests/public_api/public_api_resolve_mapping_statuses.robot +++ b/tests/robot-tests/tests/public_api/public_api_resolve_mapping_statuses.robot @@ -12,7 +12,6 @@ Suite Teardown user closes the browser Test Setup fail test fast if required - *** Variables *** ${PUBLICATION_NAME}= UI tests - Public API - resolve mapping statuses %{RUN_IDENTIFIER} ${RELEASE_NAME}= Financial year 3000-01 @@ -20,10 +19,9 @@ ${SUBJECT_NAME_1}= UI test subject 1 ${SUBJECT_NAME_2}= UI test subject 2 - *** Test Cases *** Create publication and release - ${PUBLICATION_ID}= user creates test publication via api ${PUBLICATION_NAME} + ${PUBLICATION_ID}= user creates test publication via api ${PUBLICATION_NAME} user creates test release via api ${PUBLICATION_ID} FY 3000 user navigates to draft release page from dashboard ${PUBLICATION_NAME} ... ${RELEASE_NAME} @@ -33,7 +31,8 @@ Verify release summary user verifies release summary Financial year 3000-01 Accredited official statistics Upload datafile - user uploads subject and waits until complete ${SUBJECT_NAME_1} absence_school.csv absence_school.meta.csv ${PUBLIC_API_FILES_DIR} + user uploads subject and waits until complete ${SUBJECT_NAME_1} absence_school.csv absence_school.meta.csv + ... ${PUBLIC_API_FILES_DIR} Add data guidance to subjects user clicks link Data and files @@ -58,7 +57,7 @@ Create 1st API dataset user clicks button Create API data set ${modal}= user waits until modal is visible Create a new API data set - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_1} + user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_1} user clicks button Confirm new API data set user waits until page finishes loading @@ -83,7 +82,8 @@ Create a second draft release via api user creates release from publication page ${PUBLICATION_NAME} Academic year 3010 Upload subject to second release - user uploads subject and waits until complete ${SUBJECT_NAME_2} absence_school_major_manual.csv absence_school_major_manual.meta.csv ${PUBLIC_API_FILES_DIR} + user uploads subject and waits until complete ${SUBJECT_NAME_2} absence_school_major_manual.csv + ... absence_school_major_manual.meta.csv ${PUBLIC_API_FILES_DIR} Add data guidance to second release user clicks link Data and files @@ -109,10 +109,11 @@ Create a different version of an API dataset(Major version) user waits until h3 is visible Current live API data sets user checks table column heading contains 1 1 Version xpath://table[@data-testid="live-api-data-sets"] - user clicks button in table cell 1 3 Create new version xpath://table[@data-testid="live-api-data-sets"] + user clicks button in table cell 1 3 Create new version + ... xpath://table[@data-testid="live-api-data-sets"] ${modal}= user waits until modal is visible Create a new API data set version - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_2} + user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_2} user clicks button Confirm new data set version user waits until page finishes loading @@ -120,23 +121,32 @@ Create a different version of an API dataset(Major version) Validate the summary contents inside the 'draft version details' table user waits until h3 is visible Draft version details - user waits until element contains css:dl[data-testid="draft-version-summary"] > div:nth-of-type(1) > dt + dd v2.0 %{WAIT_LONG} - user waits until element contains css:dl[data-testid="draft-version-summary"] > div:nth-of-type(2) > dt + dd Action required %{WAIT_LONG} + user waits until element contains css:dl[data-testid="draft-version-summary"] > div:nth-of-type(1) > dt + dd + ... v2.0 %{WAIT_LONG} + user waits until element contains css:dl[data-testid="draft-version-summary"] > div:nth-of-type(2) > dt + dd + ... Action required %{WAIT_LONG} ${mapping_status}= get text css:dl[data-testid="draft-version-summary"] > div:nth-of-type(2) > dt + dd should be equal as strings ${mapping_status} Action required Validate the version task statuses inside the 'Draft version task' section user waits until h3 is visible Draft version tasks - user waits until element contains css:div[data-testid="draft-version-tasks"] li:nth-child(1) a Map locations %{WAIT_LONG} - user waits until element contains css:div[data-testid="draft-version-tasks"] li:nth-child(2) a Map filters %{WAIT_LONG} - - user waits until element contains css:div[data-testid="draft-version-tasks"] li:nth-child(1) div[id="map-locations-task-status"] Incomplete %{WAIT_LONG} - user waits until element contains css:div[data-testid="draft-version-tasks"] li:nth-child(2) div[id="map-filters-task-status"] Incomplete %{WAIT_LONG} + user waits until element contains css:div[data-testid="draft-version-tasks"] li:nth-child(1) a Map locations + ... %{WAIT_LONG} + user waits until element contains css:div[data-testid="draft-version-tasks"] li:nth-child(2) a Map filters + ... %{WAIT_LONG} + + user waits until element contains + ... css:div[data-testid="draft-version-tasks"] li:nth-child(1) div[id="map-locations-task-status"] Incomplete + ... %{WAIT_LONG} + user waits until element contains + ... css:div[data-testid="draft-version-tasks"] li:nth-child(2) div[id="map-filters-task-status"] Incomplete + ... %{WAIT_LONG} User clicks on Map locations link user clicks link Map locations - user waits until h3 is visible Locations not found in new data set - user waits until element contains xpath://table[@data-testid='mappable-table-region']/caption//strong[1] 1 unmapped location %{WAIT_LONG} + user waits until h3 is visible Locations not found in new data set + user waits until element contains xpath://table[@data-testid='mappable-table-region']/caption//strong[1] + ... 1 unmapped location %{WAIT_LONG} Validate the 'unmapped location' notification banner user waits until h2 is visible Action required @@ -147,31 +157,31 @@ Validate the row headings and its contents in the 'Regions' section user checks table column heading contains 1 2 New data set user checks table column heading contains 1 3 Type - user checks table column heading contains 1 4 Actions + user checks table column heading contains 1 4 Actions user checks table cell contains 1 1 Yorkshire and The Humber user checks table cell contains 1 2 Unmapped user checks table cell contains 1 3 N/A User edits location mapping - user clicks button in table cell 1 4 Edit + user clicks button in table cell 1 4 Edit ${modal}= user waits until modal is visible Map existing location - user clicks radio Yorkshire + user clicks radio Yorkshire user clicks button Update location mapping user waits until modal is not visible Map existing location Verify mapping changes - user waits until element contains xpath://table[@data-testid='mappable-table-region']/caption//strong[1] 1 mapped location %{WAIT_LONG} + user waits until element contains xpath://table[@data-testid='mappable-table-region']/caption//strong[1] + ... 1 mapped location %{WAIT_LONG} Validate the row headings and its contents in the 'Regions' section(after mapping) - user waits until h3 is visible Locations not found in new data set user checks table column heading contains 1 1 Current data set user checks table column heading contains 1 2 New data set user checks table column heading contains 1 3 Type - user checks table column heading contains 1 4 Actions + user checks table column heading contains 1 4 Actions user checks table cell contains 1 1 Yorkshire and The Humber user checks table cell contains 1 2 Yorkshire @@ -182,15 +192,20 @@ Validate the row headings and its contents in the 'Regions' section(after mappin Validate the version status of location task user waits until h3 is visible Draft version tasks - user waits until element contains css:div[data-testid="draft-version-tasks"] li:nth-child(1) a Map locations %{WAIT_LONG} - user waits until element contains css:div[data-testid="draft-version-tasks"] li:nth-child(2) a Map filters %{WAIT_LONG} + user waits until element contains css:div[data-testid="draft-version-tasks"] li:nth-child(1) a Map locations + ... %{WAIT_LONG} + user waits until element contains css:div[data-testid="draft-version-tasks"] li:nth-child(2) a Map filters + ... %{WAIT_LONG} - user waits until element contains css:div[data-testid="draft-version-tasks"] li:nth-child(1) div[id="map-locations-task-status"] Complete %{WAIT_LONG} + user waits until element contains + ... css:div[data-testid="draft-version-tasks"] li:nth-child(1) div[id="map-locations-task-status"] Complete + ... %{WAIT_LONG} User clicks on Map filters link user clicks link Map filters - user waits until h3 is visible Filter options not found in new data set - user waits until element contains xpath://table[@data-testid='mappable-table-school_type']/caption//strong[1] 1 unmapped filter option %{WAIT_LONG} + user waits until h3 is visible Filter options not found in new data set + user waits until element contains xpath://table[@data-testid='mappable-table-school_type']/caption//strong[1] + ... 1 unmapped filter option %{WAIT_LONG} Validate the 'unmapped filter option' notification banner user waits until h2 is visible Action required @@ -201,22 +216,23 @@ Validate the row headings and its contents in the 'filter options' section user checks table column heading contains 1 2 New data set user checks table column heading contains 1 3 Type - user checks table column heading contains 1 4 Actions + user checks table column heading contains 1 4 Actions user checks table cell contains 1 1 Total user checks table cell contains 1 2 Unmapped user checks table cell contains 1 3 N/A User edits filter mapping - user clicks button in table cell 1 4 Edit + user clicks button in table cell 1 4 Edit ${modal}= user waits until modal is visible Map existing filter option - user clicks radio State-funded primary and secondary + user clicks radio State-funded primary and secondary user clicks button Update filter option mapping user waits until modal is not visible Map existing location Verify mapping changes - user waits until element contains xpath://table[@data-testid='mappable-table-school_type']/caption//strong[1] 1 mapped filter option %{WAIT_LONG} + user waits until element contains xpath://table[@data-testid='mappable-table-school_type']/caption//strong[1] + ... 1 mapped filter option %{WAIT_LONG} Validate the row headings and its contents in the 'filters options' section(after mapping) user waits until h3 is visible Filter options not found in new data set @@ -224,7 +240,7 @@ Validate the row headings and its contents in the 'filters options' section(afte user checks table column heading contains 1 2 New data set user checks table column heading contains 1 3 Type - user checks table column heading contains 1 4 Actions + user checks table column heading contains 1 4 Actions user checks table cell contains 1 1 Total user checks table cell contains 1 2 State-funded primary and secondary @@ -240,21 +256,22 @@ Confirm finalization of this API data set version User navigates to 'changelog and guidance notes' page and update relevant details in it user clicks link by index View changelog and guidance notes 1 - user waits until page contains API data set changelog + user waits until page contains API data set changelog - user enters text into element css:textarea[id="guidanceNotesForm-notes"] public guidance notes + user enters text into element css:textarea[id="guidanceNotesForm-notes"] + ... Content for the public guidance notes user clicks button Save public guidance notes - user waits until page contains public guidance notes + user waits until page contains Content for the public guidance notes user clicks link Back to API data set details User clicks on 'View preview token log' link inside the 'Draft version details' section user clicks link by index View changelog and guidance notes 2 Validate the contents in the 'API dataset changelog' page. - user waits until page contains API data set changelog + user waits until page contains API data set changelog - user waits until page contains public guidance notes + user waits until page contains Content for the public guidance notes user clicks link Back to API data set details Add headline text block to Content page @@ -280,13 +297,15 @@ Search with 2nd API dataset user waits until page finishes loading user clicks radio Newest - ${API_DATASET_STATUS_VALUE}= set variable li[data-testid="data-set-file-summary-UI test subject 2"]:nth-of-type(1) [data-testid="Status-value"] strong:nth-of-type(1) - user checks contents inside the cell value This is the latest data css:${API_DATASET_STATUS_VALUE} + ${API_DATASET_STATUS_VALUE}= set variable + ... li[data-testid="data-set-file-summary-UI test subject 2"]:nth-of-type(1) [data-testid="Status-value"] strong:nth-of-type(1) + user checks contents inside the cell value This is the latest data css:${API_DATASET_STATUS_VALUE} user checks page contains link ${SUBJECT_NAME_2} user checks list item contains testid:data-set-file-list 1 ${SUBJECT_NAME_2} User clicks on 2nd API dataset link + capture large screenshot user clicks link by index ${SUBJECT_NAME_2} user waits until page finishes loading @@ -299,16 +318,20 @@ User checks relevant headings exist on API dataset details page user waits until h2 is visible Using this data user waits until h2 is visible API data set quick start user waits until h2 is visible API data set version history - -User verifies the headings and contents in 'API version history' section - user checks table column heading contains 1 1 Version css:section[id="apiVersionHistory"] - user checks table column heading contains 1 2 Release css:section[id="apiVersionHistory"] - user checks table column heading contains 1 3 Status css:section[id="apiVersionHistory"] - - user checks table cell contains 1 1 1.1 (current) xpath://section[@id="apiVersionHistory"] - user checks table cell contains 1 2 Academic year 3010/11 xpath://section[@id="apiVersionHistory"] - user checks table cell contains 1 3 Published xpath://section[@id="apiVersionHistory"] - - user checks table cell contains 2 1 1.0 xpath://section[@id="apiVersionHistory"] - user checks table cell contains 2 2 Financial year 3000-01 xpath://section[@id="apiVersionHistory"] - user checks table cell contains 2 3 Published xpath://section[@id="apiVersionHistory"] + user waits until h2 is visible API data set changelog + +User verifies the public data guidance in the 'API data set changelog' section + user waits until element contains testid:public-guidance-notes Content for the public guidance notes + +## EES-5560 - commented out any further testing until EES-5559 is resolved. +## User verifies the major changes in the 'API data set changelog' section +## user waits until h3 is visible Major changes for version 1.1 +## ${major_changes_section}= get child element id:apiChangelog testid:major-changes +## user checks element contains ${major_changes_section} This version introduces major breaking changes +## ${deleted_indicators_section}= user checks element contains child element ${major_changes_section} +## ... testid:deleted-indicators +## user checks element contains ${deleted_indicators_section} Enrolments +## user checks element contains ${deleted_indicators_section} Number of authorised sessions +## user checks element contains ${deleted_indicators_section} Number of possible sessions +## user checks element contains ${deleted_indicators_section} Number of unauthorised sessions +## user checks element contains ${deleted_indicators_section} Percentage of unauthorised sessions From 4603d2b890da7e854373bdc24ac90db86510cb4a Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 11 Oct 2024 15:18:29 +0100 Subject: [PATCH 72/80] EES-5560 - added UI tests for minor changes in the changelog --- ...al.csv => absence_school_minor_manual.csv} | 0 ...v => absence_school_minor_manual.meta.csv} | 0 tests/robot-tests/tests/libs/common.robot | 10 +++ ...s.robot => public_api_minor_changes.robot} | 73 ++++++++++++++----- 4 files changed, 63 insertions(+), 20 deletions(-) rename tests/robot-tests/tests/files/public-api-data-files/{absence_school_major_manual.csv => absence_school_minor_manual.csv} (100%) rename tests/robot-tests/tests/files/public-api-data-files/{absence_school_major_manual.meta.csv => absence_school_minor_manual.meta.csv} (100%) rename tests/robot-tests/tests/public_api/{public_api_resolve_mapping_statuses.robot => public_api_minor_changes.robot} (84%) diff --git a/tests/robot-tests/tests/files/public-api-data-files/absence_school_major_manual.csv b/tests/robot-tests/tests/files/public-api-data-files/absence_school_minor_manual.csv similarity index 100% rename from tests/robot-tests/tests/files/public-api-data-files/absence_school_major_manual.csv rename to tests/robot-tests/tests/files/public-api-data-files/absence_school_minor_manual.csv diff --git a/tests/robot-tests/tests/files/public-api-data-files/absence_school_major_manual.meta.csv b/tests/robot-tests/tests/files/public-api-data-files/absence_school_minor_manual.meta.csv similarity index 100% rename from tests/robot-tests/tests/files/public-api-data-files/absence_school_major_manual.meta.csv rename to tests/robot-tests/tests/files/public-api-data-files/absence_school_minor_manual.meta.csv diff --git a/tests/robot-tests/tests/libs/common.robot b/tests/robot-tests/tests/libs/common.robot index d024fcd14a3..6c942d07fac 100644 --- a/tests/robot-tests/tests/libs/common.robot +++ b/tests/robot-tests/tests/libs/common.robot @@ -636,6 +636,16 @@ user waits until h3 is not visible ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until element is not visible xpath://h3[${text_matcher}] ${wait} +user waits until h4 is visible + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user waits until element is visible xpath://h4[${text_matcher}] ${wait} + +user waits until h4 is not visible + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + ${text_matcher}= get xpath text matcher ${text} ${exact_match} + user waits until element is not visible xpath://h4[${text_matcher}] ${wait} + user waits until legend is visible [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} ${text_matcher}= get xpath text matcher ${text} ${exact_match} diff --git a/tests/robot-tests/tests/public_api/public_api_resolve_mapping_statuses.robot b/tests/robot-tests/tests/public_api/public_api_minor_changes.robot similarity index 84% rename from tests/robot-tests/tests/public_api/public_api_resolve_mapping_statuses.robot rename to tests/robot-tests/tests/public_api/public_api_minor_changes.robot index 1d699564b86..4b169b8272f 100644 --- a/tests/robot-tests/tests/public_api/public_api_resolve_mapping_statuses.robot +++ b/tests/robot-tests/tests/public_api/public_api_minor_changes.robot @@ -47,7 +47,6 @@ Add data guidance to subjects user enters text into data guidance data file content editor ${SUBJECT_NAME_1} ... ${SUBJECT_NAME_1} Main guidance content -Save data guidance user clicks button Save guidance Create 1st API dataset @@ -82,8 +81,8 @@ Create a second draft release via api user creates release from publication page ${PUBLICATION_NAME} Academic year 3010 Upload subject to second release - user uploads subject and waits until complete ${SUBJECT_NAME_2} absence_school_major_manual.csv - ... absence_school_major_manual.meta.csv ${PUBLIC_API_FILES_DIR} + user uploads subject and waits until complete ${SUBJECT_NAME_2} absence_school_minor_manual.csv + ... absence_school_minor_manual.meta.csv ${PUBLIC_API_FILES_DIR} Add data guidance to second release user clicks link Data and files @@ -98,7 +97,6 @@ Add data guidance to second release user enters text into data guidance data file content editor ${SUBJECT_NAME_2} ... ${SUBJECT_NAME_2} Main guidance content -Save data guidance user clicks button Save guidance Create a different version of an API dataset(Major version) @@ -171,7 +169,7 @@ User edits location mapping user clicks button Update location mapping user waits until modal is not visible Map existing location -Verify mapping changes +Verify location mapping changes user waits until element contains xpath://table[@data-testid='mappable-table-region']/caption//strong[1] ... 1 mapped location %{WAIT_LONG} @@ -230,7 +228,7 @@ User edits filter mapping user clicks button Update filter option mapping user waits until modal is not visible Map existing location -Verify mapping changes +Verify filter mapping changes user waits until element contains xpath://table[@data-testid='mappable-table-school_type']/caption//strong[1] ... 1 mapped filter option %{WAIT_LONG} @@ -274,7 +272,7 @@ Validate the contents in the 'API dataset changelog' page. user waits until page contains Content for the public guidance notes user clicks link Back to API data set details -Add headline text block to Content page +Add headline text block to Content page for the second release user navigates to content page ${PUBLICATION_NAME} user adds headlines text block user adds content to headlines text block Headline text block text @@ -305,7 +303,6 @@ Search with 2nd API dataset user checks list item contains testid:data-set-file-list 1 ${SUBJECT_NAME_2} User clicks on 2nd API dataset link - capture large screenshot user clicks link by index ${SUBJECT_NAME_2} user waits until page finishes loading @@ -323,15 +320,51 @@ User checks relevant headings exist on API dataset details page User verifies the public data guidance in the 'API data set changelog' section user waits until element contains testid:public-guidance-notes Content for the public guidance notes -## EES-5560 - commented out any further testing until EES-5559 is resolved. -## User verifies the major changes in the 'API data set changelog' section -## user waits until h3 is visible Major changes for version 1.1 -## ${major_changes_section}= get child element id:apiChangelog testid:major-changes -## user checks element contains ${major_changes_section} This version introduces major breaking changes -## ${deleted_indicators_section}= user checks element contains child element ${major_changes_section} -## ... testid:deleted-indicators -## user checks element contains ${deleted_indicators_section} Enrolments -## user checks element contains ${deleted_indicators_section} Number of authorised sessions -## user checks element contains ${deleted_indicators_section} Number of possible sessions -## user checks element contains ${deleted_indicators_section} Number of unauthorised sessions -## user checks element contains ${deleted_indicators_section} Percentage of unauthorised sessions +User verifies minor changes in the 'API data set changelog' section + user waits until h3 is visible Minor changes for version 1.1 + ${minor_changes_section}= get child element id:apiChangelog testid:minor-changes + + ${school_types_filter}= user checks changelog section contains updated filter ${minor_changes_section} + ... School type + ${updated_school_types_total}= user checks changed facet contains option ${school_types_filter} Total + user checks changed option contains description ${updated_school_types_total} + ... label changed to: State-funded primary and secondary + user checks changed option contains description ${updated_school_types_total} no longer an aggregate + + ${updated_regional_options}= user checks changelog section contains updated location level + ... ${minor_changes_section} Regional + ${updated_yorkshire_and_humber}= user checks changed facet contains option ${updated_regional_options} + ... Yorkshire and The Humber + user checks changed option contains description ${updated_yorkshire_and_humber} label changed to: Yorkshire + + +*** Keywords *** +user checks changelog section contains updated filter + [Arguments] + ... ${changes_section} + ... ${filter_label} + ${filter}= user checks element contains child element ${changes_section} + ... xpath://div[starts-with(@data-testid, "updated-filterOptions")][h4[contains(., "Updated ${filter_label} filter options")]] + RETURN ${filter} + +user checks changelog section contains updated location level + [Arguments] + ... ${changes_section} + ... ${level_label} + ${location_level}= user checks element contains child element ${changes_section} + ... xpath://div[starts-with(@data-testid, "updated-locationOptions")][h4[contains(., "Updated ${level_label} location options")]] + RETURN ${location_level} + +user checks changed facet contains option + [Arguments] + ... ${changed_facet} + ... ${option_label} + ${option}= user checks element contains child element ${changed_facet} + ... xpath://li[@data-testid="updated-item" and contains(., "${option_label}")] + RETURN ${option} + +user checks changed option contains description + [Arguments] + ... ${changed_option} + ... ${description} + user checks element contains ${changed_option} ${description} From 6d0299554bb50263f39609c87eb3ebc99b83d475 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 11 Oct 2024 16:03:36 +0100 Subject: [PATCH 73/80] EES-5560 - adding tests for major mapping changes. Amended fail-fast code to fail the "Record failing suite" keyword to auto-expand it in the report. --- .../absence_school_major_auto.csv | 217 +++++++++++++++ .../absence_school_major_auto.meta.csv | 8 + tests/robot-tests/tests/libs/common.robot | 12 +- tests/robot-tests/tests/libs/fail_fast.py | 1 + tests/robot-tests/tests/libs/utilities.py | 6 + .../public_api_major_auto_changes.robot | 253 ++++++++++++++++++ ... => public_api_minor_manual_changes.robot} | 105 +++++--- 7 files changed, 558 insertions(+), 44 deletions(-) create mode 100644 tests/robot-tests/tests/files/public-api-data-files/absence_school_major_auto.csv create mode 100644 tests/robot-tests/tests/files/public-api-data-files/absence_school_major_auto.meta.csv create mode 100644 tests/robot-tests/tests/public_api/public_api_major_auto_changes.robot rename tests/robot-tests/tests/public_api/{public_api_minor_changes.robot => public_api_minor_manual_changes.robot} (76%) diff --git a/tests/robot-tests/tests/files/public-api-data-files/absence_school_major_auto.csv b/tests/robot-tests/tests/files/public-api-data-files/absence_school_major_auto.csv new file mode 100644 index 00000000000..ce2146b5faa --- /dev/null +++ b/tests/robot-tests/tests/files/public-api-data-files/absence_school_major_auto.csv @@ -0,0 +1,217 @@ +time_period,time_identifier,geographic_level,country_code,country_name,region_code,region_name,old_la_code,new_la_code,la_name,school_urn,school_laestab,school_name,academy_type,ncyear,enrolments,sess_possible,sess_authorised,sess_unauthorised,sess_unauthorised_percent +202021,Academic year,National,E92000001,England,,,,,,,,,,Year 4,930365,8380405,4410042,36349,10.0359 +202021,Academic year,National,E92000001,England,,,,,,,,,,Year 6,390233,2895910,2734525,418548,12.8344 +202021,Academic year,National,E92000001,England,,,,,,,,,,Year 8,966035,3669102,4767788,12507,9.4158 +202021,Academic year,National,E92000001,England,,,,,,,,,,Year 10,687704,8880914,3426082,16844,10.9951 +202021,Academic year,National,E92000001,England,,,,,,,,,,Year 4,233870,4666313,1794452,164845,7.9822 +202021,Academic year,National,E92000001,England,,,,,,,,,,Year 6,510682,608287,3047386,276594,7.806 +202021,Academic year,National,E92000001,England,,,,,,,,,,Year 8,114560,1018241,4071886,105461,3.6011 +202021,Academic year,National,E92000001,England,,,,,,,,,,Year 10,496128,8738376,3932498,276495,10.7961 +202021,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 4,748965,7281611,394560,254132,6.2354 +202021,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 6,819402,4571123,292963,79165,14.8326 +202021,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 8,999598,2688774,953422,202885,2.5727 +202021,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 10,863196,5417065,516529,70706,7.8286 +202021,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 4,960185,6838156,460125,452731,2.9259 +202021,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 6,415706,2949928,2711122,443827,1.7428 +202021,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 8,498680,7171811,3213217,134325,4.1664 +202021,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 10,706374,5427979,2488176,248703,0.9058 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 4,643309,2783829,442519,360329,9.0429 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 6,406128,3922516,529884,228791,9.5496 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 8,954008,2510684,1058785,159578,10.3886 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 10,890581,4446119,4096651,6031,5.4532 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 4,863407,6514957,4870020,470418,2.8281 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 6,134998,1099421,351548,206551,12.3929 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 8,881018,9926047,4934983,417201,5.3926 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 10,939893,502110,2088017,401671,14.2018 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,141973,3702039,Hoyland Springwood Primary School,Primary sponsor led academy,Year 4,296352,7368540,1002108,285622,3.9876 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,141973,3702039,Hoyland Springwood Primary School,Primary sponsor led academy,Year 6,222884,6082113,360617,103563,6.2636 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,106653,3704027,Penistone Grammar School,,Year 8,661641,1779059,1354689,125448,3.8439 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,106653,3704027,Penistone Grammar School,,Year 10,206716,3239290,4648058,389642,12.3234 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 4,996491,8002818,4635376,213122,1.6915 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 6,846370,2152920,2828812,338402,7.6658 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 8,122971,2345267,4613665,214274,12.1115 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 10,219610,545275,1728142,129069,13.2427 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 4,338336,8025420,3599138,249791,8.2206 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 6,12515,3001050,3315415,239804,9.0924 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 8,454021,802504,3520216,373236,3.111 +202021,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 10,48297,4774788,1558126,88665,4.0493 +202021,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,145374,3732341,Greenhill Primary School,,Year 4,193938,3822743,699491,21990,1.2297 +202021,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,145374,3732341,Greenhill Primary School,,Year 6,420838,646484,3226295,252707,2.5504 +202021,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,140821,3734008,Newfield Secondary School,Secondary sponsor led academy,Year 8,425445,6702005,3926588,19445,2.0738 +202021,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,140821,3734008,Newfield Secondary School,Secondary sponsor led academy,Year 10,178144,1389676,4679175,151593,5.4475 +202021,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 4,636969,3414663,3283314,318214,1.2402 +202021,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 6,17520,5494804,366873,266062,7.4774 +202021,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 8,817310,4511712,2233435,43576,7.9209 +202021,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 10,46096,4687214,862914,43010,10.8394 +202021,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 4,794394,1963532,564394,90260,8.1949 +202021,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 6,200199,1526080,2819473,407340,11.3028 +202021,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 8,454627,8951304,1559449,327305,4.3795 +202021,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 10,933426,2761869,1918148,185997,5.1204 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 4,991280,5127375,1163515,388622,2.175 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 6,436370,4420177,1343905,412913,7.4523 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 8,224856,47538,1958476,337417,11.5369 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 10,906144,3803424,864910,168235,4.6508 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 4,435305,3679657,2183483,176088,3.8776 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 6,742448,3740647,2876036,96880,8.604 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 8,941614,1936128,3212126,57332,14.045 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 10,969606,5896296,526533,236361,8.2235 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,101269,3022014,Colindale Primary School,,Year 4,10329,5168495,4252690,162288,4.9802 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,101269,3022014,Colindale Primary School,,Year 6,900808,3281802,4018891,402870,2.0455 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,135507,3026906,Wren Academy Finchley,Secondary sponsor led academy,Year 8,655344,6237122,3417277,99042,3.7976 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,135507,3026906,Wren Academy Finchley,Secondary sponsor led academy,Year 10,789368,5583863,4304947,456541,5.7804 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 4,500911,9586188,578180,21607,3.3617 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 6,477675,9610295,3190487,392393,12.0798 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 8,216141,5952523,4784805,486676,12.6185 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 10,423972,61023,2440931,332898,8.5528 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 4,154631,5934750,716423,263603,0.8892 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 6,48296,3079505,4666548,373958,10.2297 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 8,708802,8694392,3370782,109291,13.5185 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 10,498079,7008979,406947,380686,6.9813 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,102579,3142032,King Athelstan Primary School,,Year 4,924172,5229817,461921,255056,10.6752 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,102579,3142032,King Athelstan Primary School,,Year 6,828609,7613912,3268057,168198,5.1854 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,141862,3144001,The Kingston Academy,Secondary free school,Year 8,485419,3294823,313667,408852,13.7387 +202021,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,141862,3144001,The Kingston Academy,Secondary free school,Year 10,948391,8779565,2607975,135041,12.8666 +202122,Academic year,National,E92000001,England,,,,,,,,,,Year 4,263424,8721586,4167742,383478,5.0415 +202122,Academic year,National,E92000001,England,,,,,,,,,,Year 6,198408,5264285,2710311,494993,5.0129 +202122,Academic year,National,E92000001,England,,,,,,,,,,Year 8,484230,1614115,3534379,87050,6.8986 +202122,Academic year,National,E92000001,England,,,,,,,,,,Year 10,155301,4190557,3231686,207486,5.8223 +202122,Academic year,National,E92000001,England,,,,,,,,,,Year 4,611553,9635683,786941,477230,14.2482 +202122,Academic year,National,E92000001,England,,,,,,,,,,Year 6,752711,3936811,752220,157318,14.1615 +202122,Academic year,National,E92000001,England,,,,,,,,,,Year 8,1072,3365122,3565402,328941,0.7839 +202122,Academic year,National,E92000001,England,,,,,,,,,,Year 10,408184,8799602,22441,236508,6.3432 +202122,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 4,610330,7640992,3720172,167078,12.0133 +202122,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 6,890478,5184289,26656,197176,6.2351 +202122,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 8,649107,4218939,3217805,343102,13.4956 +202122,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 10,63988,3357225,4901965,332298,8.693 +202122,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 4,343528,6477506,3420795,94339,9.0596 +202122,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 6,974891,3272204,3420489,457219,1.7708 +202122,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 8,809181,8606505,3497757,284217,4.8312 +202122,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 10,267865,3977718,4007129,279593,14.0394 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 4,375793,5942431,186830,124526,9.5212 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 6,732072,1971603,4584099,297972,7.3678 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 8,262655,7628392,370082,207129,5.1999 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 10,683791,569658,2478482,281011,4.9364 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 4,977317,9595365,1839374,414659,12.3688 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 6,211650,5928607,1691306,386536,13.4202 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 8,163986,9675857,661047,250482,1.0812 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 10,130938,1788491,1248863,40642,0.505 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,141973,3702039,Hoyland Springwood Primary School,Primary sponsor led academy,Year 4,291660,4552605,1441830,421128,11.8714 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,141973,3702039,Hoyland Springwood Primary School,Primary sponsor led academy,Year 6,691504,8180301,3458032,11936,7.1623 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,106653,3704027,Penistone Grammar School,,Year 8,87349,6608543,1831731,38458,2.2866 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,106653,3704027,Penistone Grammar School,,Year 10,335593,4145901,430493,219634,3.8077 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 4,65626,8941189,2778792,40825,12.4985 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 6,446000,8180991,4241359,160601,4.396 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 8,672736,2069839,1789330,25253,13.106 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 10,710863,4131441,412250,234342,12.7331 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 4,735373,1481670,3320679,249061,7.5083 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 6,315922,1657744,1998333,182112,12.0564 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 8,322703,7598586,4782452,34528,4.1753 +202122,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 10,807678,777312,2469553,493181,6.233 +202122,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,145374,3732341,Greenhill Primary School,,Year 4,389695,1193807,3889011,292288,10.5549 +202122,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,145374,3732341,Greenhill Primary School,,Year 6,298947,3650745,4370595,53048,14.3211 +202122,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,140821,3734008,Newfield Secondary School,Secondary sponsor led academy,Year 8,29393,2940163,460442,337613,7.0733 +202122,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,140821,3734008,Newfield Secondary School,Secondary sponsor led academy,Year 10,752009,5840033,262396,356628,7.901 +202122,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 4,934820,1670483,4183571,467206,7.207 +202122,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 6,10142,7126488,1679362,418191,14.0816 +202122,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 8,444973,5610555,4967515,359239,9.3129 +202122,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 10,855721,8989241,2794631,444771,3.8782 +202122,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 4,38547,1265455,4454444,127033,5.9484 +202122,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 6,217205,4114641,824011,477351,7.0324 +202122,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 8,474073,9860956,4416348,426962,13.956 +202122,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 10,38647,8375228,700134,343753,0.4792 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 4,165477,3982999,2881117,494074,5.1466 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 6,322047,7661435,4262799,97475,12.4722 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 8,18416,6851799,2025239,270083,7.1532 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 10,940546,4035799,1299367,17693,14.8835 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 4,27697,9793497,4056200,22348,11.8263 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 6,967464,8034305,1390806,421037,4.7727 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 8,932041,2054142,4961909,486625,8.1061 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 10,200464,8732143,2099505,150975,2.4462 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,101269,3022014,Colindale Primary School,,Year 4,466618,4331684,2336745,474374,3.2321 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,101269,3022014,Colindale Primary School,,Year 6,51873,7318963,4091886,382787,3.9492 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,135507,3026906,Wren Academy Finchley,Secondary sponsor led academy,Year 8,324513,9629567,2090853,165020,13.2842 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,135507,3026906,Wren Academy Finchley,Secondary sponsor led academy,Year 10,12135,418821,3617819,471557,1.9472 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 4,429828,2541098,4189699,191836,4.6809 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 6,800267,821476,4743561,380527,5.9484 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 8,277672,1725609,1984888,476621,3.1818 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 10,938510,8382254,4570224,373427,5.9204 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 4,563379,9886857,211791,138796,8.1423 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 6,262520,5231953,1946565,190296,4.8397 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 8,319358,3860000,2182522,407747,11.8051 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 10,958386,9724817,3819718,327783,11.8484 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,102579,3142032,King Athelstan Primary School,,Year 4,825224,8846588,1975540,131067,5.4658 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,102579,3142032,King Athelstan Primary School,,Year 6,922630,6219569,737030,39992,3.7797 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,141862,3144001,The Kingston Academy,Secondary free school,Year 8,887815,8905360,511940,110254,8.985 +202122,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,141862,3144001,The Kingston Academy,Secondary free school,Year 10,826025,3168076,1954888,14999,3.7619 +202223,Academic year,National,E92000001,England,,,,,,,,,,Year 4,219863,1211324,2461223,227036,6.8479 +202223,Academic year,National,E92000001,England,,,,,,,,,,Year 6,326899,9400873,3596596,84154,6.4159 +202223,Academic year,National,E92000001,England,,,,,,,,,,Year 8,714610,561356,2907605,433541,5.2496 +202223,Academic year,National,E92000001,England,,,,,,,,,,Year 10,383602,1803285,287864,2883,0.4685 +202223,Academic year,National,E92000001,England,,,,,,,,,,Year 4,654884,3220663,4379854,207878,4.939 +202223,Academic year,National,E92000001,England,,,,,,,,,,Year 6,235647,9247378,3142734,451761,3.1564 +202223,Academic year,National,E92000001,England,,,,,,,,,,Year 8,625982,1024199,387768,48184,1.1214 +202223,Academic year,National,E92000001,England,,,,,,,,,,Year 10,703403,7341429,3350111,90245,6.689 +202223,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 4,356747,6400821,178815,353079,7.2406 +202223,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 6,478301,6941924,3385453,137969,14.2775 +202223,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 8,634983,9628460,3931328,65536,3.7371 +202223,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 10,540587,6663635,2179562,323178,1.3373 +202223,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 4,783767,9416803,3342067,61772,10.9315 +202223,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 6,818474,6521510,3326180,148818,10.1909 +202223,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 8,935461,7564846,874297,88073,9.912 +202223,Academic year,Regional,E92000001,England,E12000003,Yorkshire and The Humber,,,,,,,,Year 10,340526,2628965,2973134,435451,2.0839 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 4,362381,1027328,577798,217717,12.313 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 6,448489,2859195,602823,176301,3.8162 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 8,654686,7414000,339649,162343,11.753 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 10,540640,9354927,2039373,98018,14.8837 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 4,316064,4874652,2347907,416236,5.8798 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 6,314436,3348975,886840,62684,9.9694 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 8,634778,1878275,2963673,212801,14.4038 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,,,,,Year 10,198910,6941070,4072452,296754,7.803 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,141973,3702039,Hoyland Springwood Primary School,Primary sponsor led academy,Year 4,294564,6547062,4745554,25788,4.6605 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,141973,3702039,Hoyland Springwood Primary School,Primary sponsor led academy,Year 6,123104,7290192,4578941,19677,5.1375 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,106653,3704027,Penistone Grammar School,,Year 8,628305,7084823,3811123,453695,8.4844 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,370,E08000016,Barnsley,106653,3704027,Penistone Grammar School,,Year 10,659044,2634214,4070198,97405,0.2416 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 4,357353,1739000,4425257,285842,12.5505 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 6,724898,1043207,389401,473394,14.1792 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 8,216117,7168883,4215280,78439,13.679 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 10,957960,9934276,1638073,147204,10.8101 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 4,563389,9121298,3303786,403465,10.987 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 6,624827,2354255,3052199,48720,12.3966 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 8,228238,7828108,2950344,452062,3.2874 +202223,Academic year,Local authority,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,,,,,Year 10,467375,9278504,422419,244619,11.617 +202223,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,145374,3732341,Greenhill Primary School,,Year 4,296027,2759449,4467649,78337,7.2387 +202223,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,145374,3732341,Greenhill Primary School,,Year 6,98921,9008990,2602662,484586,6.1155 +202223,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,140821,3734008,Newfield Secondary School,Secondary sponsor led academy,Year 8,929132,182167,4080556,483888,8.1479 +202223,Academic year,School,E92000001,England,E12000003,Yorkshire and The Humber,373,E08000019,Sheffield,140821,3734008,Newfield Secondary School,Secondary sponsor led academy,Year 10,751028,1449807,175843,263574,9.6395 +202223,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 4,577746,7493602,1377020,315021,7.8505 +202223,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 6,720401,3593781,3441828,186132,12.8428 +202223,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 8,470788,4851295,1733986,360850,5.5305 +202223,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 10,903536,8331786,1760720,215307,10.8352 +202223,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 4,834635,2581506,2352884,153955,13.0697 +202223,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 6,854011,18306,4206838,121002,13.2874 +202223,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 8,304325,2900642,690657,217905,1.9265 +202223,Academic year,Regional,E92000001,England,E13000002,Outer London,,,,,,,,Year 10,965673,3633069,1760280,40032,1.8564 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 4,288238,9847925,4084552,444552,11.8302 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 6,775782,7953739,787260,180127,8.7798 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 8,783460,233298,1339828,148935,7.5016 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 10,524941,7991277,1172325,363994,10.5637 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 4,930007,9048500,128025,460991,0.9231 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 6,929516,9022992,4420240,350476,7.6263 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 8,535524,5970083,3697539,328489,4.6321 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,,,,,Year 10,435263,4919212,1055600,422276,12.6506 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,101269,3022014,Colindale Primary School,,Year 4,519041,7906994,4405937,89041,4.744 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,101269,3022014,Colindale Primary School,,Year 6,674329,776260,1342817,241868,9.1741 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,135507,3026906,Wren Academy Finchley,Secondary sponsor led academy,Year 8,455256,921184,4608260,20074,8.3842 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,302,E09000003,Barnet,135507,3026906,Wren Academy Finchley,Secondary sponsor led academy,Year 10,135692,7302541,324393,244258,7.7954 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 4,303770,670289,4766068,486141,13.9153 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 6,37346,4344143,1516845,387614,3.5746 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 8,903110,9011174,417857,433299,9.0392 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 10,100966,3931562,4064499,301608,13.4953 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 4,750557,1399665,270713,209296,3.9268 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 6,147524,53494,2877891,10248,2.3631 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 8,92708,5795664,2225558,165760,0.5731 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,,,,,Year 10,620300,8365110,938239,103917,14.8688 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,102579,3142032,King Athelstan Primary School,,Year 4,695463,951512,2881136,224021,1.7931 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,102579,3142032,King Athelstan Primary School,,Year 6,57424,7954628,3860012,134564,11.4722 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,141862,3144001,The Kingston Academy,Secondary free school,Year 8,304011,8070409,1297358,26349,2.1772 +202223,Academic year,Local authority,E92000001,England,E13000002,Outer London,314,E09000021 / E09000027,Kingston upon Thames / Richmond upon Thames,141862,3144001,The Kingston Academy,Secondary free school,Year 10,422137,145499,2199762,452887,8.9538 diff --git a/tests/robot-tests/tests/files/public-api-data-files/absence_school_major_auto.meta.csv b/tests/robot-tests/tests/files/public-api-data-files/absence_school_major_auto.meta.csv new file mode 100644 index 00000000000..a996d310689 --- /dev/null +++ b/tests/robot-tests/tests/files/public-api-data-files/absence_school_major_auto.meta.csv @@ -0,0 +1,8 @@ +col_name,col_type,label,indicator_grouping,indicator_unit,indicator_dp,filter_hint,filter_grouping_column +enrolments,Indicator,Enrolments,Headline absence fields,,0,, +sess_possible,Indicator,Number of possible sessions,Headline absence fields,,0,, +sess_authorised,Indicator,Number of authorised sessions,Headline absence fields,,0,, +sess_unauthorised,Indicator,Number of unauthorised sessions,Headline absence fields,,0,, +sess_unauthorised_percent,Indicator,Percentage of unauthorised sessions,Headline absence fields,%,2,, +academy_type,Filter,Academy type,,,,"Only applicable for academies, otherwise no value", +ncyear,Filter,National Curriculum year,,,,Ranges from years 1 to 11, diff --git a/tests/robot-tests/tests/libs/common.robot b/tests/robot-tests/tests/libs/common.robot index 6c942d07fac..24aa8c209c2 100644 --- a/tests/robot-tests/tests/libs/common.robot +++ b/tests/robot-tests/tests/libs/common.robot @@ -681,12 +681,20 @@ user checks textarea contains user checks summary list contains [Arguments] ${term} ${description} ${parent}=css:body ${wait}=${timeout} user waits until parent contains element ${parent} - ... xpath:.//dl//dt[contains(text(), "${term}")]/following-sibling::dd[contains(., "${description}")] + ... xpath:.//dt[contains(text(), "${term}")]/following-sibling::dd[contains(., "${description}")] ... %{WAIT_MEDIUM} ${element}= get child element ${parent} - ... xpath:.//dl//dt[contains(text(), "${term}")]/following-sibling::dd[contains(., "${description}")] + ... xpath:.//dt[contains(text(), "${term}")]/following-sibling::dd[contains(., "${description}")] user waits until element is visible ${element} %{WAIT_LONG} +user checks summary list does not contain + [Arguments] ${term} ${description} ${parent}=css:body ${wait}=${timeout} + user waits until parent contains element ${parent} + ... xpath:.//dt[contains(text(), "${term}")] + ... %{WAIT_MEDIUM} + user waits until parent does not contain element ${parent} + ... xpath:.//dt[contains(text(), "${term}")]/following-sibling::dd[contains(., "${description}")] + user checks select contains x options [Arguments] ${locator} ${num} ${options}= get list items ${locator} diff --git a/tests/robot-tests/tests/libs/fail_fast.py b/tests/robot-tests/tests/libs/fail_fast.py index b3fac71af57..6f72a594891 100644 --- a/tests/robot-tests/tests/libs/fail_fast.py +++ b/tests/robot-tests/tests/libs/fail_fast.py @@ -27,6 +27,7 @@ def record_test_failure(): visual.capture_screenshot() visual.capture_large_screenshot() _capture_html() + _raise_assertion_error("Recorded test failure") if BuiltIn().get_variable_value("${prompt_to_continue_on_failure}") == "1": _prompt_to_continue() diff --git a/tests/robot-tests/tests/libs/utilities.py b/tests/robot-tests/tests/libs/utilities.py index ea450e63541..f92a1dd7702 100644 --- a/tests/robot-tests/tests/libs/utilities.py +++ b/tests/robot-tests/tests/libs/utilities.py @@ -43,10 +43,16 @@ def _find_by_testid(parent_locator: object, criteria: str, tag: str, constraints return get_child_elements(parent_locator, f'css:[data-testid="{criteria}"]') + def _find_by_text(parent_locator: object, criteria: str, tag: str, constraints: dict) -> list: + parent_locator = _normalize_parent_locator(parent_locator) + + return get_child_elements(parent_locator, f'xpath:.//*[contains(., "{criteria}")]') + # Register locator strategies element_finder().register("label", _find_by_label, persist=True) element_finder().register("testid", _find_by_testid, persist=True) + element_finder().register("text", _find_by_text, persist=True) utilities_init.initialised = True diff --git a/tests/robot-tests/tests/public_api/public_api_major_auto_changes.robot b/tests/robot-tests/tests/public_api/public_api_major_auto_changes.robot new file mode 100644 index 00000000000..d06b4a60bd1 --- /dev/null +++ b/tests/robot-tests/tests/public_api/public_api_major_auto_changes.robot @@ -0,0 +1,253 @@ +*** Settings *** +Library ../libs/admin_api.py +Resource ../libs/admin-common.robot +Resource ../libs/admin/manage-content-common.robot +Resource ../libs/public-common.robot +Resource ../libs/public-api-common.robot + +Force Tags Admin Local Dev AltersData + +Suite Setup user signs in as bau1 +Suite Teardown user closes the browser +Test Setup fail test fast if required +Test Teardown Run Keyword If Test Failed record test failure + + +*** Variables *** +${PUBLICATION_NAME}= UI tests - Public API - major auto changes %{RUN_IDENTIFIER} +${RELEASE_1_NAME}= Financial year 3000-01 +${RELEASE_2_NAME}= Academic year 3010/11 +${SUBJECT_1_NAME}= UI tests - Public API - major auto changes - subject 1 - %{RUN_IDENTIFIER} +${SUBJECT_2_NAME}= UI tests - Public API - major auto changes - subject 2 - %{RUN_IDENTIFIER} + + +*** Test Cases *** +Create publication and release + ${PUBLICATION_ID}= user creates test publication via api ${PUBLICATION_NAME} + user creates test release via api ${PUBLICATION_ID} FY 3000 + user navigates to draft release page from dashboard ${PUBLICATION_NAME} + ... ${RELEASE_1_NAME} + +Verify release summary + user checks page contains element xpath://li/a[text()="Summary" and contains(@aria-current, 'page')] + user verifies release summary Financial year 3000-01 Accredited official statistics + +Upload datafile + user uploads subject and waits until complete ${SUBJECT_1_NAME} absence_school.csv absence_school.meta.csv + ... ${PUBLIC_API_FILES_DIR} + +Add data guidance to subjects + user clicks link Data and files + user waits until h2 is visible Add data file to release + + user clicks link Data guidance + user waits until h2 is visible Public data guidance + + user waits until page contains element id:dataGuidance-dataFiles + user waits until page contains accordion section ${SUBJECT_1_NAME} + + user enters text into data guidance data file content editor ${SUBJECT_1_NAME} + ... ${SUBJECT_1_NAME} Main guidance content + + user clicks button Save guidance + +Create 1st API dataset + user scrolls to the top of the page + user clicks link API data sets + user waits until h2 is visible API data sets + + user clicks button Create API data set + ${modal}= user waits until modal is visible Create a new API data set + user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_1_NAME} + user clicks button Confirm new API data set + + user waits until page finishes loading + user waits until modal is not visible Create a new API data set + +User waits until the 1st API dataset status changes to 'Ready' + user waits until h3 is visible Draft version details + wait until keyword succeeds 10x %{WAIT_SMALL}s Verify status of API Datasets Ready + +Add headline text block to Content page + user clicks link Back to API data sets + user navigates to content page ${PUBLICATION_NAME} + user adds headlines text block + user adds content to headlines text block Headline text block text + +Approve first release + user clicks link Sign off + user approves release for immediate publication + +Create a second draft release via api + user navigates to publication page from dashboard ${PUBLICATION_NAME} + user creates release from publication page ${PUBLICATION_NAME} Academic year 3010 + +# In this new subject, the "School type" filter has been removed entirely. + +Upload subject to second release + user uploads subject and waits until complete ${SUBJECT_2_NAME} absence_school_major_auto.csv + ... absence_school_major_auto.meta.csv ${PUBLIC_API_FILES_DIR} + +Add data guidance to second release + user clicks link Data and files + user waits until h2 is visible Add data file to release + + user clicks link Data guidance + user waits until h2 is visible Public data guidance + + user waits until page contains element id:dataGuidance-dataFiles + user waits until page contains accordion section ${SUBJECT_2_NAME} + + user enters text into data guidance data file content editor ${SUBJECT_2_NAME} + ... ${SUBJECT_2_NAME} Main guidance content + + user clicks button Save guidance + +Create a different version of an API dataset with major changes + user scrolls to the top of the page + user clicks link API data sets + user waits until h2 is visible API data sets + + user waits until h3 is visible Current live API data sets + + user checks table column heading contains 1 1 Version xpath://table[@data-testid="live-api-data-sets"] + user clicks button in table cell 1 3 Create new version + ... xpath://table[@data-testid="live-api-data-sets"] + + ${modal}= user waits until modal is visible Create a new API data set version + user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_2_NAME} + user clicks button Confirm new data set version + + user waits until page finishes loading + user waits until modal is not visible Create a new API data set version + +Validate the summary contents inside the 'Latest live version details' table + user waits until h3 is visible Draft version details + user checks summary list contains Version v1.0 parent=id:live-version-summary + user checks summary list contains Status Published parent=id:live-version-summary + user checks summary list contains Release ${RELEASE_1_NAME} parent=id:live-version-summary + user checks summary list contains Data set file ${SUBJECT_1_NAME} parent=id:live-version-summary + user checks summary list contains Geographic levels Local authority, National, Regional, School + ... parent=id:live-version-summary + user checks summary list contains Time periods 2020/21 to 2022/23 parent=id:live-version-summary + user checks summary list contains Indicators Enrolments parent=id:live-version-summary + user checks summary list contains Indicators more indicators parent=id:live-version-summary + user checks summary list contains Filters Academy type parent=id:live-version-summary + user checks summary list contains Filters School type parent=id:live-version-summary + user checks summary list contains Actions View live data set (opens in new tab) + ... parent=id:live-version-summary + +Validate the summary contents inside the 'draft version details' table + user waits until h3 is visible Draft version details + user checks summary list contains Version v2.0 parent=id:draft-version-summary wait=%{WAIT_LONG} + user checks summary list contains Status Action required parent=id:draft-version-summary + ... wait=%{WAIT_LONG} + user checks summary list contains Release ${RELEASE_2_NAME} parent=id:draft-version-summary + user checks summary list contains Data set file ${SUBJECT_2_NAME} parent=id:draft-version-summary + user checks summary list contains Geographic levels Local authority, National, Regional, School + ... parent=id:draft-version-summary + user checks summary list contains Time periods 2020/21 to 2022/23 parent=id:draft-version-summary + user checks summary list contains Indicators Enrolments parent=id:draft-version-summary + user checks summary list contains Indicators more indicators parent=id:draft-version-summary + user checks summary list contains Filters Academy type parent=id:draft-version-summary + user checks summary list does not contain Filters School type parent=id:draft-version-summary + user checks summary list contains Actions Remove draft version parent=id:draft-version-summary + +Validate that location and filter mappings are complete + user waits until h3 is visible Draft version tasks + user waits until parent contains element testid:map-locations-task link:Map locations + user waits until parent contains element id:map-locations-task-status text:Complete + user waits until parent contains element testid:map-filters-task link:Map filters + user waits until parent contains element id:map-filters-task-status text:Complete + +Confirm finalization of this API data set version + user clicks button Finalise this data set version + user waits for caches to expire + user waits until h2 is visible Mappings finalised + user waits until page contains Draft API data set version is ready to be published + +User navigates to 'changelog and guidance notes' page and update relevant details in it + user clicks link by index View changelog and guidance notes 1 + user waits until page contains API data set changelog + + user enters text into element css:textarea[id="guidanceNotesForm-notes"] + ... Content for the public guidance notes + user clicks button Save public guidance notes + + user waits until page contains Content for the public guidance notes + user clicks link Back to API data set details + +User clicks on 'View preview token log' link inside the 'Draft version details' section + user clicks link by index View changelog and guidance notes 2 + +Validate the contents in the 'API dataset changelog' page. + user waits until page contains API data set changelog + + user waits until page contains Content for the public guidance notes + user clicks link Back to API data set details + +Add headline text block to Content page for the second release + user navigates to content page ${PUBLICATION_NAME} + user adds headlines text block + user adds content to headlines text block Headline text block text + +Approve second release + user clicks link Sign off + user approves release for immediate publication + +Verify newly published release is on Find Statistics page + user checks publication is on find statistics page ${PUBLICATION_NAME} + +User navigates to data catalogue page + user navigates to data catalogue page on public frontend + +Search with 2nd API dataset + user clicks element id:searchForm-search + user presses keys ${PUBLICATION_NAME} + user clicks radio API data sets only + + user waits until page finishes loading + user clicks radio Newest + + user checks summary list contains Status This is the latest data + ... parent=testid:data-set-file-summary-${SUBJECT_2_NAME} + user checks summary list contains Status Available by API + ... parent=testid:data-set-file-summary-${SUBJECT_2_NAME} + user checks page contains link ${SUBJECT_2_NAME} + +User clicks on 2nd API dataset link + user clicks link ${SUBJECT_2_NAME} + user waits until page finishes loading + user waits until h1 is visible ${SUBJECT_2_NAME} + +User checks relevant headings exist on API dataset details page + user waits until h2 is visible Data set details + user waits until h2 is visible Data set preview + user waits until h2 is visible Variables in this data set + user waits until h2 is visible Using this data + user waits until h2 is visible API data set quick start + user waits until h2 is visible API data set version history + user waits until h2 is visible API data set changelog + +User verifies the public data guidance in the 'API data set changelog' section + user waits until element contains testid:public-guidance-notes Content for the public guidance notes + +User verifies major changes in the 'API data set changelog' section + user waits until h3 is visible Major changes for version 2.0 + ${major_changes_Section}= get child element id:apiChangelog testid:major-changes + + ${school_types_filter}= user checks changelog section contains deleted filter ${major_changes_Section} + ... School type + + user checks page does not contain element testid:minor-changes + + +*** Keywords *** +user checks changelog section contains deleted filter + [Arguments] + ... ${changes_section} + ... ${filter_label} + ${deleted_filters}= user checks element contains child element ${changes_section} + ... xpath://div[@data-testid="deleted-filters"] + user checks element contains child element ${deleted_filters} + ... xpath://li[@data-testid="deleted-item" and contains(., "${filter_label}")] diff --git a/tests/robot-tests/tests/public_api/public_api_minor_changes.robot b/tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot similarity index 76% rename from tests/robot-tests/tests/public_api/public_api_minor_changes.robot rename to tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot index 4b169b8272f..1017adaa2fe 100644 --- a/tests/robot-tests/tests/public_api/public_api_minor_changes.robot +++ b/tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot @@ -10,13 +10,15 @@ Force Tags Admin Local Dev AltersData Suite Setup user signs in as bau1 Suite Teardown user closes the browser Test Setup fail test fast if required +Test Teardown Run Keyword If Test Failed record test failure *** Variables *** -${PUBLICATION_NAME}= UI tests - Public API - resolve mapping statuses %{RUN_IDENTIFIER} -${RELEASE_NAME}= Financial year 3000-01 -${SUBJECT_NAME_1}= UI test subject 1 -${SUBJECT_NAME_2}= UI test subject 2 +${PUBLICATION_NAME}= UI tests - Public API - minor manual changes %{RUN_IDENTIFIER} +${RELEASE_1_NAME}= Financial year 3000-01 +${RELEASE_2_NAME}= Academic year 3010/11 +${SUBJECT_1_NAME}= UI tests - Public API - minor manual changes - subject 1 - %{RUN_IDENTIFIER} +${SUBJECT_2_NAME}= UI tests - Public API - minor manual changes - subject 2 - %{RUN_IDENTIFIER} *** Test Cases *** @@ -24,14 +26,14 @@ Create publication and release ${PUBLICATION_ID}= user creates test publication via api ${PUBLICATION_NAME} user creates test release via api ${PUBLICATION_ID} FY 3000 user navigates to draft release page from dashboard ${PUBLICATION_NAME} - ... ${RELEASE_NAME} + ... ${RELEASE_1_NAME} Verify release summary user checks page contains element xpath://li/a[text()="Summary" and contains(@aria-current, 'page')] user verifies release summary Financial year 3000-01 Accredited official statistics Upload datafile - user uploads subject and waits until complete ${SUBJECT_NAME_1} absence_school.csv absence_school.meta.csv + user uploads subject and waits until complete ${SUBJECT_1_NAME} absence_school.csv absence_school.meta.csv ... ${PUBLIC_API_FILES_DIR} Add data guidance to subjects @@ -42,10 +44,10 @@ Add data guidance to subjects user waits until h2 is visible Public data guidance user waits until page contains element id:dataGuidance-dataFiles - user waits until page contains accordion section ${SUBJECT_NAME_1} + user waits until page contains accordion section ${SUBJECT_1_NAME} - user enters text into data guidance data file content editor ${SUBJECT_NAME_1} - ... ${SUBJECT_NAME_1} Main guidance content + user enters text into data guidance data file content editor ${SUBJECT_1_NAME} + ... ${SUBJECT_1_NAME} Main guidance content user clicks button Save guidance @@ -56,7 +58,7 @@ Create 1st API dataset user clicks button Create API data set ${modal}= user waits until modal is visible Create a new API data set - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_1} + user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_1_NAME} user clicks button Confirm new API data set user waits until page finishes loading @@ -80,8 +82,12 @@ Create a second draft release via api user navigates to publication page from dashboard ${PUBLICATION_NAME} user creates release from publication page ${PUBLICATION_NAME} Academic year 3010 +# In this new subject, the "School type" filter option "Total" is being updated to "State-funded primary and secondary", +# and is no longer marked as an aggregate option. The location option "Yorkshire and the Humber" is being renamed to +# "Yorkshire". + Upload subject to second release - user uploads subject and waits until complete ${SUBJECT_NAME_2} absence_school_minor_manual.csv + user uploads subject and waits until complete ${SUBJECT_2_NAME} absence_school_minor_manual.csv ... absence_school_minor_manual.meta.csv ${PUBLIC_API_FILES_DIR} Add data guidance to second release @@ -92,14 +98,14 @@ Add data guidance to second release user waits until h2 is visible Public data guidance user waits until page contains element id:dataGuidance-dataFiles - user waits until page contains accordion section ${SUBJECT_NAME_2} + user waits until page contains accordion section ${SUBJECT_2_NAME} - user enters text into data guidance data file content editor ${SUBJECT_NAME_2} - ... ${SUBJECT_NAME_2} Main guidance content + user enters text into data guidance data file content editor ${SUBJECT_2_NAME} + ... ${SUBJECT_2_NAME} Main guidance content user clicks button Save guidance -Create a different version of an API dataset(Major version) +Create a different version of an API dataset with minor changes user scrolls to the top of the page user clicks link API data sets user waits until h2 is visible API data sets @@ -111,34 +117,48 @@ Create a different version of an API dataset(Major version) ... xpath://table[@data-testid="live-api-data-sets"] ${modal}= user waits until modal is visible Create a new API data set version - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_NAME_2} + user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_2_NAME} user clicks button Confirm new data set version user waits until page finishes loading user waits until modal is not visible Create a new API data set version +Validate the summary contents inside the 'Latest live version details' table + user waits until h3 is visible Draft version details + user checks summary list contains Version v1.0 parent=id:live-version-summary + user checks summary list contains Status Published parent=id:live-version-summary + user checks summary list contains Release ${RELEASE_1_NAME} parent=id:live-version-summary + user checks summary list contains Data set file ${SUBJECT_1_NAME} parent=id:live-version-summary + user checks summary list contains Geographic levels Local authority, National, Regional, School + ... parent=id:live-version-summary + user checks summary list contains Time periods 2020/21 to 2022/23 parent=id:live-version-summary + user checks summary list contains Indicators Enrolments parent=id:live-version-summary + user checks summary list contains Indicators more indicators parent=id:live-version-summary + user checks summary list contains Filters Academy type parent=id:live-version-summary + user checks summary list contains Actions View live data set (opens in new tab) + ... parent=id:live-version-summary + Validate the summary contents inside the 'draft version details' table user waits until h3 is visible Draft version details - user waits until element contains css:dl[data-testid="draft-version-summary"] > div:nth-of-type(1) > dt + dd - ... v2.0 %{WAIT_LONG} - user waits until element contains css:dl[data-testid="draft-version-summary"] > div:nth-of-type(2) > dt + dd - ... Action required %{WAIT_LONG} - ${mapping_status}= get text css:dl[data-testid="draft-version-summary"] > div:nth-of-type(2) > dt + dd - should be equal as strings ${mapping_status} Action required + user checks summary list contains Version v2.0 parent=id:draft-version-summary wait=%{WAIT_LONG} + user checks summary list contains Status Action required parent=id:draft-version-summary + ... wait=%{WAIT_LONG} + user checks summary list contains Release ${RELEASE_2_NAME} parent=id:draft-version-summary + user checks summary list contains Data set file ${SUBJECT_2_NAME} parent=id:draft-version-summary + user checks summary list contains Geographic levels Local authority, National, Regional, School + ... parent=id:draft-version-summary + user checks summary list contains Time periods 2020/21 to 2022/23 parent=id:draft-version-summary + user checks summary list contains Indicators Enrolments parent=id:draft-version-summary + user checks summary list contains Indicators more indicators parent=id:draft-version-summary + user checks summary list contains Filters Academy type parent=id:draft-version-summary + user checks summary list contains Actions Remove draft version parent=id:draft-version-summary Validate the version task statuses inside the 'Draft version task' section user waits until h3 is visible Draft version tasks - user waits until element contains css:div[data-testid="draft-version-tasks"] li:nth-child(1) a Map locations - ... %{WAIT_LONG} - user waits until element contains css:div[data-testid="draft-version-tasks"] li:nth-child(2) a Map filters - ... %{WAIT_LONG} - - user waits until element contains - ... css:div[data-testid="draft-version-tasks"] li:nth-child(1) div[id="map-locations-task-status"] Incomplete - ... %{WAIT_LONG} - user waits until element contains - ... css:div[data-testid="draft-version-tasks"] li:nth-child(2) div[id="map-filters-task-status"] Incomplete - ... %{WAIT_LONG} + user waits until parent contains element testid:map-locations-task link:Map locations + user waits until parent contains element id:map-locations-task-status text:Incomplete + user waits until parent contains element testid:map-filters-task link:Map filters + user waits until parent contains element id:map-filters-task-status text:Incomplete User clicks on Map locations link user clicks link Map locations @@ -295,18 +315,16 @@ Search with 2nd API dataset user waits until page finishes loading user clicks radio Newest - ${API_DATASET_STATUS_VALUE}= set variable - ... li[data-testid="data-set-file-summary-UI test subject 2"]:nth-of-type(1) [data-testid="Status-value"] strong:nth-of-type(1) - user checks contents inside the cell value This is the latest data css:${API_DATASET_STATUS_VALUE} - user checks page contains link ${SUBJECT_NAME_2} - - user checks list item contains testid:data-set-file-list 1 ${SUBJECT_NAME_2} + user checks summary list contains Status This is the latest data + ... parent=testid:data-set-file-summary-${SUBJECT_2_NAME} + user checks summary list contains Status Available by API + ... parent=testid:data-set-file-summary-${SUBJECT_2_NAME} + user checks page contains link ${SUBJECT_2_NAME} User clicks on 2nd API dataset link - user clicks link by index ${SUBJECT_NAME_2} + user clicks link ${SUBJECT_2_NAME} user waits until page finishes loading - - user waits until h1 is visible ${SUBJECT_NAME_2} + user waits until h1 is visible ${SUBJECT_2_NAME} User checks relevant headings exist on API dataset details page user waits until h2 is visible Data set details @@ -337,6 +355,9 @@ User verifies minor changes in the 'API data set changelog' section ... Yorkshire and The Humber user checks changed option contains description ${updated_yorkshire_and_humber} label changed to: Yorkshire +User verifies no major changes are present + user checks page does not contain element testid:major-changes + *** Keywords *** user checks changelog section contains updated filter From c13afe6dc1b494256921266cbd61c41bb9e8719e Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 14 Oct 2024 15:07:39 +0100 Subject: [PATCH 74/80] EES-5560 - corrected "Total runs" cound in UI Test Slack message. Fixed flaky last step of Subject Reordering test --- .../tests/admin_and_public/bau/subject_reordering.robot | 7 ++++--- tests/robot-tests/tests/libs/slack.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/robot-tests/tests/admin_and_public/bau/subject_reordering.robot b/tests/robot-tests/tests/admin_and_public/bau/subject_reordering.robot index 326aeece16f..2d8f974e21a 100644 --- a/tests/robot-tests/tests/admin_and_public/bau/subject_reordering.robot +++ b/tests/robot-tests/tests/admin_and_public/bau/subject_reordering.robot @@ -218,8 +218,9 @@ Check subject order in data catalogue user navigates to data catalogue page on public frontend user wait for option to be available and select it css:select[id="filters-form-theme"] %{TEST_THEME_NAME} - user wait for option to be available and select it css:select[id="filters-form-publication"] ${PUBLICATION_NAME} - sleep 1 # wait a moment to wait for release filter options to get updated + user wait for option to be available and select it css:select[id="filters-form-publication"] + ... ${PUBLICATION_NAME} + sleep 1 # wait a moment to wait for release filter options to get updated user wait for option to be available and select it css:select[id="filters-form-release"] ${RELEASE_NAME} user waits until page contains Download all 4 data sets (ZIP) @@ -232,7 +233,7 @@ Check subject order in data catalogue ... locator=xpath://*[@data-testid="data-set-file-list"]/li/h4 Check subject order in data guidance - user navigates to public find statistics page + user checks publication is on find statistics page ${PUBLICATION_NAME} user clicks link ${PUBLICATION_NAME} user waits until h1 is visible ${PUBLICATION_NAME} %{WAIT_MEDIUM} diff --git a/tests/robot-tests/tests/libs/slack.py b/tests/robot-tests/tests/libs/slack.py index 31536dfc2b6..2c2595ccbde 100644 --- a/tests/robot-tests/tests/libs/slack.py +++ b/tests/robot-tests/tests/libs/slack.py @@ -22,7 +22,7 @@ def __init__(self): self.client = WebClient(token=self.slack_app_token) - def _build_test_results_attachments(self, env: str, suites_ran: str, suites_failed: [], run_index: int): + def _build_test_results_attachments(self, env: str, suites_ran: str, suites_failed: [], number_of_test_runs: int): with open(f"{PATH}{os.sep}output.xml", "rb") as report: contents = report.read() @@ -51,7 +51,7 @@ def _build_test_results_attachments(self, env: str, suites_ran: str, suites_fail "fields": [ {"type": "mrkdwn", "text": f"*Environment*\n{env}"}, {"type": "mrkdwn", "text": f"*Suite*\n{suites_ran.replace('tests/', '')}"}, - {"type": "mrkdwn", "text": f"*Total runs*\n{run_index + 1}"}, + {"type": "mrkdwn", "text": f"*Total runs*\n{number_of_test_runs}"}, {"type": "mrkdwn", "text": f"*Total test cases*\n{total_tests_count}"}, {"type": "mrkdwn", "text": f"*Passed test cases*\n{passed_tests}"}, {"type": "mrkdwn", "text": f"*Failed test cases*\n{failed_tests}"}, From ca8419dfe6450b9f529e515c8e54de8b0f69138d Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 14 Oct 2024 17:24:35 +0100 Subject: [PATCH 75/80] EES-5560 - responding to PR comments. Amending case of True and False reserved words in Robot files. --- .../bau/create_data_block_with_chart.robot | 2 +- .../admin/bau/prerelease_and_amend.robot | 2 +- ...table_tool_absence_by_characteristic.robot | 2 +- .../robot-tests/tests/libs/admin-common.robot | 4 +- .../libs/admin/manage-content-common.robot | 12 +- tests/robot-tests/tests/libs/common.robot | 110 +++++++++--------- .../visual_testing/tables_and_charts.robot | 4 +- 7 files changed, 68 insertions(+), 68 deletions(-) diff --git a/tests/robot-tests/tests/admin/bau/create_data_block_with_chart.robot b/tests/robot-tests/tests/admin/bau/create_data_block_with_chart.robot index 0314c3021d5..be808fde98a 100644 --- a/tests/robot-tests/tests/admin/bau/create_data_block_with_chart.robot +++ b/tests/robot-tests/tests/admin/bau/create_data_block_with_chart.robot @@ -488,7 +488,7 @@ Add reference line user clicks button Add new line user chooses select option id:chartAxisConfiguration-major-referenceLines-position 2005 user enters text into element id:chartAxisConfiguration-major-referenceLines-label Reference line 1 - user clicks button Add exact_match=${TRUE} + user clicks button Add exact_match=${True} Validate basic line chart preview user waits until element contains line chart id:chartBuilderPreview diff --git a/tests/robot-tests/tests/admin/bau/prerelease_and_amend.robot b/tests/robot-tests/tests/admin/bau/prerelease_and_amend.robot index 2b42a1352e8..bd0b6c42f8f 100644 --- a/tests/robot-tests/tests/admin/bau/prerelease_and_amend.robot +++ b/tests/robot-tests/tests/admin/bau/prerelease_and_amend.robot @@ -142,7 +142,7 @@ Add basic release content user navigates to content page ${PUBLICATION_NAME} # FALSE to not add headline block, as we needed to add that to publish the original release - user adds basic release content ${PUBLICATION_NAME} ${FALSE} + user adds basic release content ${PUBLICATION_NAME} ${False} Add release note to amendment user clicks button Add note diff --git a/tests/robot-tests/tests/general_public/table_tool_absence_by_characteristic.robot b/tests/robot-tests/tests/general_public/table_tool_absence_by_characteristic.robot index 14792ac8799..92c9a07b598 100644 --- a/tests/robot-tests/tests/general_public/table_tool_absence_by_characteristic.robot +++ b/tests/robot-tests/tests/general_public/table_tool_absence_by_characteristic.robot @@ -137,7 +137,7 @@ Reorder Gender to be column group user clicks button Move and reorder table headers # Column group needs to be inside the viewport user scrolls to element xpath://button[text()="Update and view reordered table"] - user clicks button Move testId:rowGroups-0 exact_match=${TRUE} + user clicks button Move testId:rowGroups-0 exact_match=${True} user clicks button Move Characteristic to columns testId:rowGroups-0 user clicks button Done testId:columnGroups-1 diff --git a/tests/robot-tests/tests/libs/admin-common.robot b/tests/robot-tests/tests/libs/admin-common.robot index 45a30572429..1655e2f645d 100644 --- a/tests/robot-tests/tests/libs/admin-common.robot +++ b/tests/robot-tests/tests/libs/admin-common.robot @@ -724,14 +724,14 @@ user changes methodology status to Approved user clicks element id:methodologyStatusForm-status-Approved user enters text into element id:methodologyStatusForm-latestInternalReleaseNote Approved by UI tests user clicks element id:methodologyStatusForm-publishingStrategy-${publishing_strategy} - IF ${is_publishing_strategy_with_release} is ${TRUE} + IF ${is_publishing_strategy_with_release} is ${True} user waits until element is enabled css:[name="withReleaseId"] user chooses select option css:[name="withReleaseId"] ${with_release} END user clicks button Update status user waits until h2 is visible Sign off user checks summary list contains Status Approved - IF ${is_publishing_strategy_with_release} is ${TRUE} + IF ${is_publishing_strategy_with_release} is ${True} user checks summary list contains When to publish With a specific release user checks summary list contains Publish with release ${with_release} ELSE diff --git a/tests/robot-tests/tests/libs/admin/manage-content-common.robot b/tests/robot-tests/tests/libs/admin/manage-content-common.robot index 3a2ea2d4546..6f4aa9ca887 100644 --- a/tests/robot-tests/tests/libs/admin/manage-content-common.robot +++ b/tests/robot-tests/tests/libs/admin/manage-content-common.robot @@ -27,7 +27,7 @@ user navigates to content page user waits until page finishes loading user adds basic release content - [Arguments] ${publication} ${add_headlines_block}=${TRUE} + [Arguments] ${publication} ${add_headlines_block}=${True} user adds summary text block user adds content to summary text block Test summary text for ${publication} @@ -186,9 +186,9 @@ user chooses and embeds data block [Arguments] ... ${datablock_name} user chooses select option css:select[name="selectedDataBlock"] ${datablock_name} - user waits until button is enabled Embed %{WAIT_SMALL} exact_match=${TRUE} - user clicks button Embed exact_match=${TRUE} - user waits until page does not contain button Embed %{WAIT_MEDIUM} exact_match=${TRUE} + user waits until button is enabled Embed %{WAIT_SMALL} exact_match=${True} + user clicks button Embed exact_match=${True} + user waits until page does not contain button Embed %{WAIT_MEDIUM} exact_match=${True} user waits until page finishes loading user opens nth editable accordion section @@ -374,7 +374,7 @@ user adds content to accordion section text block ${block}= user starts editing accordion section text block ${section_name} ${block_num} ${parent} - IF "${append}" == "${FALSE}" + IF "${append}" == "${False}" user presses keys CTRL+a user presses keys BACKSPACE ELSE @@ -475,7 +475,7 @@ user adds image to accordion section text block with retry sleep 5 user scrolls up 100 wait until keyword succeeds ${timeout} %{WAIT_SMALL} sec user clicks element - ... xpath://div[@title="Insert paragraph after block"] + ... css:[title="Insert paragraph after block"] # wait for the API to save the image and for the src attribute to be updated before continuing user waits until parent contains element ${block} diff --git a/tests/robot-tests/tests/libs/common.robot b/tests/robot-tests/tests/libs/common.robot index 24aa8c209c2..b0793d21729 100644 --- a/tests/robot-tests/tests/libs/common.robot +++ b/tests/robot-tests/tests/libs/common.robot @@ -220,25 +220,25 @@ user waits until element contains testid user waits until parent contains element ${element} css:[data-testid="${testid}"] timeout=${wait} user waits until page contains accordion section - [Arguments] ${section_title} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${section_title} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${section_title} ${exact_match} user waits until page contains element ... xpath://button[@class='govuk-accordion__section-button'][.//span[${text_matcher}]] ${wait} user waits until page does not contain accordion section - [Arguments] ${section_title} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${section_title} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${section_title} ${exact_match} user waits until page does not contain element ... xpath://button[@class='govuk-accordion__section-button'][.//span[${text_matcher}]] ${wait} user verifies accordion is open - [Arguments] ${section_text} ${exact_match}=${FALSE} + [Arguments] ${section_text} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${section_text} ${exact_match} user waits until page contains element ... xpath://button[@class='govuk-accordion__section-button'][.//span[${text_matcher}] and @aria-expanded="true"] user verifies accordion is closed - [Arguments] ${section_text} ${exact_match}=${FALSE} + [Arguments] ${section_text} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${section_text} ${exact_match} user waits until page contains element ... xpath://button[@class='govuk-accordion__section-button'][.//span[${text_matcher}] and @aria-expanded="false"] @@ -248,19 +248,19 @@ user checks there are x accordion sections user waits until parent contains element ${parent} css:[data-testid="accordionSection"] count=${count} user checks accordion is in position - [Arguments] ${section_text} ${position} ${parent}=css:[data-testid="accordion"] ${exact_match}=${FALSE} + [Arguments] ${section_text} ${position} ${parent}=css:[data-testid="accordion"] ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${section_text} ${exact_match} user waits until parent contains element ${parent} ... xpath:(.//*[@data-testid="accordionSection"])[${position}]//span[${text_matcher}] user waits until accordion section contains text - [Arguments] ${section_text} ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${section_text} ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} ${section}= user gets accordion section content element ${section_text} user waits until parent contains element ${section} xpath:.//*[${text_matcher}] timeout=${wait} user gets accordion header button element - [Arguments] ${heading_text} ${parent}=css:[data-testid="accordion"] ${exact_match}=${FALSE} + [Arguments] ${heading_text} ${parent}=css:[data-testid="accordion"] ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${heading_text} ${exact_match} ${button}= get child element ${parent} xpath:.//button[@aria-expanded and ${text_matcher}] [Return] ${button} @@ -269,7 +269,7 @@ user opens accordion section [Arguments] ... ${heading_text} ... ${parent}=css:[data-testid="accordion"] - ... ${exact_match}=${FALSE} + ... ${exact_match}=${False} ${header_button}= user gets accordion header button element ${heading_text} ${parent} ... exact_match=${exact_match} @@ -299,7 +299,7 @@ user opens accordion section with accordion header [Return] ${accordion} user closes accordion section - [Arguments] ${heading_text} ${parent}=css:[data-testid="accordion"] ${exact_match}=${FALSE} + [Arguments] ${heading_text} ${parent}=css:[data-testid="accordion"] ${exact_match}=${False} ${header_button}= user gets accordion header button element ${heading_text} ${parent} ... exact_match=${exact_match} user closes accordion section with accordion header ${header_button} ${parent} exact_match=${exact_match} @@ -308,7 +308,7 @@ user closes accordion section with id [Arguments] ... ${id} ... ${parent}=css:[data-testid="accordion"] - ... ${exact_match}=${FALSE} + ... ${exact_match}=${False} ${header_button}= get child element ${parent} id:${id}-heading user closes accordion section with accordion header ${header_button} ${parent} exact_match=${exact_match} @@ -317,7 +317,7 @@ user closes accordion section with accordion header [Arguments] ... ${header_button} ... ${parent}=css:[data-testid="accordion"] - ... ${exact_match}=${FALSE} + ... ${exact_match}=${False} ${is_expanded}= get element attribute ${header_button} aria-expanded IF '${is_expanded}' != 'false' @@ -326,7 +326,7 @@ user closes accordion section with accordion header user checks element attribute value should be ${header_button} aria-expanded false user gets accordion section content element - [Arguments] ${heading_text} ${parent}=css:[data-testid="accordion"] ${exact_match}=${FALSE} + [Arguments] ${heading_text} ${parent}=css:[data-testid="accordion"] ${exact_match}=${False} ${header_button}= user gets accordion header button element ${heading_text} ${parent} ... exact_match=${exact_match} ${content_id}= get element attribute ${header_button} aria-controls @@ -340,7 +340,7 @@ user gets accordion section content element from heading element [Return] ${content} user scrolls to accordion section - [Arguments] ${heading_text} ${parent}=css:[data-testid="accordion"] ${exact_match}=${FALSE} + [Arguments] ${heading_text} ${parent}=css:[data-testid="accordion"] ${exact_match}=${False} ${header_button}= user gets accordion header button element ${heading_text} ${parent} ... exact_match=${exact_match} ${content}= user gets accordion section content element ${heading_text} ${parent} @@ -386,7 +386,7 @@ user checks element does not contain child element user waits until parent does not contain element ${element} ${child_element} user checks element contains - [Arguments] ${element} ${text} ${exact_match}=${FALSE} + [Arguments] ${element} ${text} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until parent contains element ${element} xpath://*[${text_matcher}] @@ -510,17 +510,17 @@ user clicks link by index user clicks element ${button} ${parent} user clicks link containing text - [Arguments] ${text} ${parent}=css:body ${exact_match}=${FALSE} + [Arguments] ${text} ${parent}=css:body ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user clicks element xpath:.//a[${text_matcher}] ${parent} user clicks button - [Arguments] ${text} ${parent}=css:body ${exact_match}=${FALSE} + [Arguments] ${text} ${parent}=css:body ${exact_match}=${False} ${button}= user gets button element ${text} ${parent} exact_match=${exact_match} user clicks element ${button} user clicks button by index - [Arguments] ${text} ${index}=1 ${parent}=css:body ${exact_match}=${FALSE} + [Arguments] ${text} ${index}=1 ${parent}=css:body ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} ${xpath}= set variable (//button[${text_matcher}])[${index}] ${button}= get webelement ${xpath} @@ -534,67 +534,67 @@ user waits until button is clickable element should be enabled xpath=//button[text()="${button_text}"] user clicks button containing text - [Arguments] ${text} ${parent}=css:body ${exact_match}=${FALSE} + [Arguments] ${text} ${parent}=css:body ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user clicks element xpath://button[${text_matcher}] ${parent} user waits until page contains button - [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until page contains element xpath://button[${text_matcher}] ... ${wait} user checks page contains button - [Arguments] ${text} ${exact_match}=${FALSE} + [Arguments] ${text} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user checks page contains element xpath://button[${text_matcher}] user checks page does not contain button - [Arguments] ${text} ${exact_match}=${FALSE} + [Arguments] ${text} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user checks page does not contain element xpath://button[${text_matcher}] user waits until page does not contain button - [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until page does not contain element ... xpath://button[${text_matcher}] ${wait} user waits until button is enabled - [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until element is enabled xpath://button[${text_matcher}] ... ${wait} user waits until parent contains button - [Arguments] ${parent} ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${parent} ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until parent contains element ${parent} ... xpath://button[${text_matcher}] ${wait} user waits until parent does not contain button - [Arguments] ${parent} ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${parent} ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until parent does not contain element ${parent} ... xpath://button[${text_matcher}] ${wait} user waits until parent does not contain - [Arguments] ${parent} ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${parent} ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until parent does not contain element ${parent} ... //button[${text_matcher}] ${wait} user gets button element - [Arguments] ${text} ${parent}=css:body ${exact_match}=${FALSE} + [Arguments] ${text} ${parent}=css:body ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until parent contains button ${parent} ${text} ${button}= get child element ${parent} xpath://button[${text_matcher}] [Return] ${button} get xpath text matcher - [Arguments] ${text} ${exact_match}=${FALSE} - IF "${exact_match}" == "${TRUE}" + [Arguments] ${text} ${exact_match}=${False} + IF "${exact_match}" == "${True}" ${expression}= Set Variable text()="${text}" ELSE ${expression}= Set Variable contains(., "${text}") @@ -602,63 +602,63 @@ get xpath text matcher RETURN ${expression} user checks page contains tag - [Arguments] ${text} ${exact_match}=${FALSE} + [Arguments] ${text} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user checks page contains element xpath://*[contains(@class, "govuk-tag")][${text_matcher}] user waits until h1 is visible - [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until element is visible xpath://h1[${text_matcher}] ${wait} user waits until h1 is not visible - [Arguments] ${text} ${wait}=%{WAIT_SMALL} ${exact_match}=${FALSE} + [Arguments] ${text} ${wait}=%{WAIT_SMALL} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until element is not visible xpath://h1[${text_matcher}] ${wait} user waits until h2 is visible - [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until element is visible xpath://h2[${text_matcher}] ${wait} user waits until h2 is not visible - [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until element is not visible xpath://h2[${text_matcher}] ${wait} user waits until h3 is visible - [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until element is visible xpath://h3[${text_matcher}] ${wait} user waits until h3 is not visible - [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until element is not visible xpath://h3[${text_matcher}] ${wait} user waits until h4 is visible - [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until element is visible xpath://h4[${text_matcher}] ${wait} user waits until h4 is not visible - [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until element is not visible xpath://h4[${text_matcher}] ${wait} user waits until legend is visible - [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until element is visible xpath://legend[${text_matcher}] ${wait} user waits until page contains title - [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until page contains element xpath://h1[@data-testid="page-title" and ${text_matcher}] ... ${wait} user waits until page contains title caption - [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until page contains element ... xpath://span[@data-testid="page-title-caption" and ${text_matcher}] ${wait} @@ -782,7 +782,7 @@ user checks page contains link [Arguments] ... ${text} ... ${parent}=css:body - ... ${exact_match}=${FALSE} + ... ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until parent contains element ${parent} ... xpath:.//a[${text_matcher}] @@ -792,13 +792,13 @@ user checks page contains link with text and url ... ${text} ... ${href} ... ${parent}=css:body - ... ${exact_match}=${FALSE} + ... ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until parent contains element ${parent} ... xpath:.//a[@href="${href}" and ${text_matcher}] user opens details dropdown - [Arguments] ${text} ${parent}=css:body ${exact_match}=${FALSE} + [Arguments] ${text} ${parent}=css:body ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until parent contains element ${parent} ... xpath:.//details/summary[${text_matcher} and @aria-expanded] %{WAIT_SMALL} @@ -813,7 +813,7 @@ user opens details dropdown [Return] ${details} user closes details dropdown - [Arguments] ${text} ${parent}=css:body ${exact_match}=${FALSE} + [Arguments] ${text} ${parent}=css:body ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until parent contains element ${parent} ... xpath:.//details/summary[contains(., "${text}") and @aria-expanded] @@ -826,7 +826,7 @@ user closes details dropdown user checks element attribute value should be ${summary} aria-expanded false user gets details content element - [Arguments] ${text} ${parent}=css:body ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${text} ${parent}=css:body ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until parent contains element ${parent} xpath:.//details/summary[${text_matcher}] ... timeout=${wait} @@ -836,17 +836,17 @@ user gets details content element [Return] ${content} user waits until page contains details dropdown - [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user waits until page contains element xpath:.//details/summary[${text_matcher}] ${wait} user checks page for details dropdown - [Arguments] ${text} ${exact_match}=${FALSE} + [Arguments] ${text} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user checks page contains element xpath:.//details/summary[${text_matcher}] user scrolls to details dropdown - [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${FALSE} + [Arguments] ${text} ${wait}=${timeout} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${text} ${exact_match} user scrolls to element xpath:.//details/summary[${text_matcher}] @@ -898,13 +898,13 @@ user checks radio is checked user checks page contains element xpath://label[text()="${label}"]/../input[@type="radio" and @checked] user checks radio in position has label - [Arguments] ${position} ${label} ${exact_match}=${FALSE} + [Arguments] ${position} ${label} ${exact_match}=${False} ${text_matcher}= get xpath text matcher ${label} ${exact_match} user checks page contains element ... xpath://*[contains(@data-testid, "Radio item for ")][${position}]//label[${text_matcher}] user clicks checkbox - [Arguments] ${label} ${exact_match}=${TRUE} + [Arguments] ${label} ${exact_match}=${True} ${text_matcher}= get xpath text matcher ${label} ${exact_match} user scrolls to element xpath://label[${text_matcher} or strong[${text_matcher}]]/../input[@type="checkbox"] user clicks element xpath://label[${text_matcher} or strong[${text_matcher}]]/../input[@type="checkbox"] @@ -915,13 +915,13 @@ user clicks checkbox by selector user clicks element ${locator} user checks checkbox is checked - [Arguments] ${label} ${exact_match}=${TRUE} + [Arguments] ${label} ${exact_match}=${True} ${text_matcher}= get xpath text matcher ${label} ${exact_match} user checks checkbox input is checked ... xpath://label[${text_matcher} or strong[${text_matcher}]]/../input[@type="checkbox"] user checks checkbox is not checked - [Arguments] ${label} ${exact_match}=${TRUE} + [Arguments] ${label} ${exact_match}=${True} ${text_matcher}= get xpath text matcher ${label} ${exact_match} user checks checkbox input is not checked ... xpath://label[${text_matcher} or strong[${text_matcher}]]/../input[@type="checkbox"] @@ -937,7 +937,7 @@ user checks checkbox input is not checked checkbox should not be selected ${selector} user checks checkbox in position has label - [Arguments] ${position} ${label} ${exact_match}=${TRUE} + [Arguments] ${position} ${label} ${exact_match}=${True} ${text_matcher}= get xpath text matcher ${label} ${exact_match} user checks page contains element ... xpath://*[contains(@data-testid,"Checkbox item for ")][${position}]//label[${text_matcher}] @@ -1069,7 +1069,7 @@ lookup or return webelement ... ${parent}=css:body ${is_webelement}= is webelement ${selector_or_webelement} - IF ${is_webelement} is ${TRUE} + IF ${is_webelement} is ${True} ${element}= set variable ${selector_or_webelement} ELSE user waits until parent contains element ${parent} ${selector_or_webelement} diff --git a/tests/robot-tests/tests/visual_testing/tables_and_charts.robot b/tests/robot-tests/tests/visual_testing/tables_and_charts.robot index 8b90ae64637..0e73f25b9fa 100644 --- a/tests/robot-tests/tests/visual_testing/tables_and_charts.robot +++ b/tests/robot-tests/tests/visual_testing/tables_and_charts.robot @@ -71,9 +71,9 @@ Check Fast Track Table ... ${SNAPSHOT_FOLDER}/${content_block.release_id}/${FAST_TRACKS_FOLDER}/${content_block.content_block_id}-table.html IF ${content_block.has_chart_config} - log content block details ${content_block} Fast Track ${TRUE} + log content block details ${content_block} Fast Track ${True} ELSE - log content block details ${content_block} Fast Track ${FALSE} + log content block details ${content_block} Fast Track ${False} END Check Content Block Table From 79c240d1bde870e17e0f7f933f5d1e9d2ffdb11b Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 14 Oct 2024 17:41:40 +0100 Subject: [PATCH 76/80] EES-5560 - responding to PR comments. Simplifying xpath selectors to be css selectors when performing simple matching on HTML attributes --- .../bau/publish_content.robot | 2 +- .../admin_and_public/bau/publish_data.robot | 2 +- .../public_api_major_auto_changes.robot | 46 +++++++-------- .../public_api_minor_manual_changes.robot | 57 +++++++++---------- 4 files changed, 51 insertions(+), 56 deletions(-) diff --git a/tests/robot-tests/tests/admin_and_public/bau/publish_content.robot b/tests/robot-tests/tests/admin_and_public/bau/publish_content.robot index 351cd7fd3ab..95f12518986 100644 --- a/tests/robot-tests/tests/admin_and_public/bau/publish_content.robot +++ b/tests/robot-tests/tests/admin_and_public/bau/publish_content.robot @@ -49,7 +49,7 @@ Add text block with link to absence glossary entry to accordion section ... ${RELEASE_CONTENT_EDITABLE_ACCORDION} ${toolbar}= get editor toolbar ${block} ${insert}= get child element parent_locator=${toolbar} - ... child_locator=xpath://button[@data-cke-tooltip-text="Insert"] + ... child_locator=css:[data-cke-tooltip-text="Insert"] user clicks element ${insert} ${button}= user gets button element Insert glossary link ${toolbar} user clicks element ${button} diff --git a/tests/robot-tests/tests/admin_and_public/bau/publish_data.robot b/tests/robot-tests/tests/admin_and_public/bau/publish_data.robot index 57f15f5e4e8..2d7aad64470 100644 --- a/tests/robot-tests/tests/admin_and_public/bau/publish_data.robot +++ b/tests/robot-tests/tests/admin_and_public/bau/publish_data.robot @@ -505,7 +505,7 @@ Add text block with link to a featured table to accordion section ... ${RELEASE_CONTENT_EDITABLE_ACCORDION} ${toolbar}= get editor toolbar ${block} ${insert}= get child element parent_locator=${toolbar} - ... child_locator=xpath://button[@data-cke-tooltip-text="Insert"] + ... child_locator=css:[data-cke-tooltip-text="Insert"] user clicks element ${insert} ${button}= user gets button element Insert featured table link ${toolbar} user clicks element ${button} diff --git a/tests/robot-tests/tests/public_api/public_api_major_auto_changes.robot b/tests/robot-tests/tests/public_api/public_api_major_auto_changes.robot index d06b4a60bd1..0f72a16c74a 100644 --- a/tests/robot-tests/tests/public_api/public_api_major_auto_changes.robot +++ b/tests/robot-tests/tests/public_api/public_api_major_auto_changes.robot @@ -123,35 +123,35 @@ Create a different version of an API dataset with major changes Validate the summary contents inside the 'Latest live version details' table user waits until h3 is visible Draft version details - user checks summary list contains Version v1.0 parent=id:live-version-summary - user checks summary list contains Status Published parent=id:live-version-summary - user checks summary list contains Release ${RELEASE_1_NAME} parent=id:live-version-summary - user checks summary list contains Data set file ${SUBJECT_1_NAME} parent=id:live-version-summary + user checks summary list contains Version v1.0 id:live-version-summary + user checks summary list contains Status Published id:live-version-summary + user checks summary list contains Release ${RELEASE_1_NAME} id:live-version-summary + user checks summary list contains Data set file ${SUBJECT_1_NAME} id:live-version-summary user checks summary list contains Geographic levels Local authority, National, Regional, School - ... parent=id:live-version-summary - user checks summary list contains Time periods 2020/21 to 2022/23 parent=id:live-version-summary - user checks summary list contains Indicators Enrolments parent=id:live-version-summary - user checks summary list contains Indicators more indicators parent=id:live-version-summary - user checks summary list contains Filters Academy type parent=id:live-version-summary - user checks summary list contains Filters School type parent=id:live-version-summary + ... id:live-version-summary + user checks summary list contains Time periods 2020/21 to 2022/23 id:live-version-summary + user checks summary list contains Indicators Enrolments id:live-version-summary + user checks summary list contains Indicators more indicators id:live-version-summary + user checks summary list contains Filters Academy type id:live-version-summary + user checks summary list contains Filters School type id:live-version-summary user checks summary list contains Actions View live data set (opens in new tab) - ... parent=id:live-version-summary + ... id:live-version-summary Validate the summary contents inside the 'draft version details' table user waits until h3 is visible Draft version details - user checks summary list contains Version v2.0 parent=id:draft-version-summary wait=%{WAIT_LONG} - user checks summary list contains Status Action required parent=id:draft-version-summary + user checks summary list contains Version v2.0 id:draft-version-summary wait=%{WAIT_LONG} + user checks summary list contains Status Action required id:draft-version-summary ... wait=%{WAIT_LONG} - user checks summary list contains Release ${RELEASE_2_NAME} parent=id:draft-version-summary - user checks summary list contains Data set file ${SUBJECT_2_NAME} parent=id:draft-version-summary + user checks summary list contains Release ${RELEASE_2_NAME} id:draft-version-summary + user checks summary list contains Data set file ${SUBJECT_2_NAME} id:draft-version-summary user checks summary list contains Geographic levels Local authority, National, Regional, School - ... parent=id:draft-version-summary - user checks summary list contains Time periods 2020/21 to 2022/23 parent=id:draft-version-summary - user checks summary list contains Indicators Enrolments parent=id:draft-version-summary - user checks summary list contains Indicators more indicators parent=id:draft-version-summary - user checks summary list contains Filters Academy type parent=id:draft-version-summary - user checks summary list does not contain Filters School type parent=id:draft-version-summary - user checks summary list contains Actions Remove draft version parent=id:draft-version-summary + ... id:draft-version-summary + user checks summary list contains Time periods 2020/21 to 2022/23 id:draft-version-summary + user checks summary list contains Indicators Enrolments id:draft-version-summary + user checks summary list contains Indicators more indicators id:draft-version-summary + user checks summary list contains Filters Academy type id:draft-version-summary + user checks summary list does not contain Filters School type id:draft-version-summary + user checks summary list contains Actions Remove draft version id:draft-version-summary Validate that location and filter mappings are complete user waits until h3 is visible Draft version tasks @@ -248,6 +248,6 @@ user checks changelog section contains deleted filter ... ${changes_section} ... ${filter_label} ${deleted_filters}= user checks element contains child element ${changes_section} - ... xpath://div[@data-testid="deleted-filters"] + ... testid:deleted-filters user checks element contains child element ${deleted_filters} ... xpath://li[@data-testid="deleted-item" and contains(., "${filter_label}")] diff --git a/tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot b/tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot index 1017adaa2fe..6ab31eb3756 100644 --- a/tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot +++ b/tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot @@ -112,9 +112,9 @@ Create a different version of an API dataset with minor changes user waits until h3 is visible Current live API data sets - user checks table column heading contains 1 1 Version xpath://table[@data-testid="live-api-data-sets"] + user checks table column heading contains 1 1 Version testid:live-api-data-sets user clicks button in table cell 1 3 Create new version - ... xpath://table[@data-testid="live-api-data-sets"] + ... testid:live-api-data-sets ${modal}= user waits until modal is visible Create a new API data set version user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_2_NAME} @@ -125,33 +125,33 @@ Create a different version of an API dataset with minor changes Validate the summary contents inside the 'Latest live version details' table user waits until h3 is visible Draft version details - user checks summary list contains Version v1.0 parent=id:live-version-summary - user checks summary list contains Status Published parent=id:live-version-summary - user checks summary list contains Release ${RELEASE_1_NAME} parent=id:live-version-summary - user checks summary list contains Data set file ${SUBJECT_1_NAME} parent=id:live-version-summary + user checks summary list contains Version v1.0 id:live-version-summary + user checks summary list contains Status Published id:live-version-summary + user checks summary list contains Release ${RELEASE_1_NAME} id:live-version-summary + user checks summary list contains Data set file ${SUBJECT_1_NAME} id:live-version-summary user checks summary list contains Geographic levels Local authority, National, Regional, School - ... parent=id:live-version-summary - user checks summary list contains Time periods 2020/21 to 2022/23 parent=id:live-version-summary - user checks summary list contains Indicators Enrolments parent=id:live-version-summary - user checks summary list contains Indicators more indicators parent=id:live-version-summary - user checks summary list contains Filters Academy type parent=id:live-version-summary + ... id:live-version-summary + user checks summary list contains Time periods 2020/21 to 2022/23 id:live-version-summary + user checks summary list contains Indicators Enrolments id:live-version-summary + user checks summary list contains Indicators more indicators id:live-version-summary + user checks summary list contains Filters Academy type id:live-version-summary user checks summary list contains Actions View live data set (opens in new tab) - ... parent=id:live-version-summary + ... id:live-version-summary Validate the summary contents inside the 'draft version details' table user waits until h3 is visible Draft version details - user checks summary list contains Version v2.0 parent=id:draft-version-summary wait=%{WAIT_LONG} - user checks summary list contains Status Action required parent=id:draft-version-summary + user checks summary list contains Version v2.0 id:draft-version-summary wait=%{WAIT_LONG} + user checks summary list contains Status Action required id:draft-version-summary ... wait=%{WAIT_LONG} - user checks summary list contains Release ${RELEASE_2_NAME} parent=id:draft-version-summary - user checks summary list contains Data set file ${SUBJECT_2_NAME} parent=id:draft-version-summary + user checks summary list contains Release ${RELEASE_2_NAME} id:draft-version-summary + user checks summary list contains Data set file ${SUBJECT_2_NAME} id:draft-version-summary user checks summary list contains Geographic levels Local authority, National, Regional, School - ... parent=id:draft-version-summary - user checks summary list contains Time periods 2020/21 to 2022/23 parent=id:draft-version-summary - user checks summary list contains Indicators Enrolments parent=id:draft-version-summary - user checks summary list contains Indicators more indicators parent=id:draft-version-summary - user checks summary list contains Filters Academy type parent=id:draft-version-summary - user checks summary list contains Actions Remove draft version parent=id:draft-version-summary + ... id:draft-version-summary + user checks summary list contains Time periods 2020/21 to 2022/23 id:draft-version-summary + user checks summary list contains Indicators Enrolments id:draft-version-summary + user checks summary list contains Indicators more indicators id:draft-version-summary + user checks summary list contains Filters Academy type id:draft-version-summary + user checks summary list contains Actions Remove draft version id:draft-version-summary Validate the version task statuses inside the 'Draft version task' section user waits until h3 is visible Draft version tasks @@ -209,15 +209,10 @@ Validate the row headings and its contents in the 'Regions' section(after mappin Validate the version status of location task user waits until h3 is visible Draft version tasks - - user waits until element contains css:div[data-testid="draft-version-tasks"] li:nth-child(1) a Map locations - ... %{WAIT_LONG} - user waits until element contains css:div[data-testid="draft-version-tasks"] li:nth-child(2) a Map filters - ... %{WAIT_LONG} - - user waits until element contains - ... css:div[data-testid="draft-version-tasks"] li:nth-child(1) div[id="map-locations-task-status"] Complete - ... %{WAIT_LONG} + user waits until parent contains element testid:map-locations-task link:Map locations + user waits until parent contains element id:map-locations-task-status text:Complete + user waits until parent contains element testid:map-filters-task link:Map filters + user waits until parent contains element id:map-filters-task-status text:Complete User clicks on Map filters link user clicks link Map filters From 5288fffb69829caf9c65718a16d53a0147833816 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 14 Oct 2024 19:18:49 +0100 Subject: [PATCH 77/80] EES-5560 - updating data set version terminology on mapping suites --- .../public_api_major_auto_changes.robot | 36 ++++++++-------- .../public_api_minor_manual_changes.robot | 42 ++++++++----------- 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/tests/robot-tests/tests/public_api/public_api_major_auto_changes.robot b/tests/robot-tests/tests/public_api/public_api_major_auto_changes.robot index 0f72a16c74a..71dc126a52c 100644 --- a/tests/robot-tests/tests/public_api/public_api_major_auto_changes.robot +++ b/tests/robot-tests/tests/public_api/public_api_major_auto_changes.robot @@ -51,22 +51,22 @@ Add data guidance to subjects user clicks button Save guidance -Create 1st API dataset +Create the initial API data set version user scrolls to the top of the page user clicks link API data sets user waits until h2 is visible API data sets user clicks button Create API data set ${modal}= user waits until modal is visible Create a new API data set - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_1_NAME} + user chooses select option name:releaseFileId ${SUBJECT_1_NAME} user clicks button Confirm new API data set user waits until page finishes loading user waits until modal is not visible Create a new API data set -User waits until the 1st API dataset status changes to 'Ready' +User waits until the initial data set version's status changes to "Ready" user waits until h3 is visible Draft version details - wait until keyword succeeds 10x %{WAIT_SMALL}s Verify status of API Datasets Ready + wait until keyword succeeds 10x %{WAIT_SMALL}s Verify status of API data sets Ready Add headline text block to Content page user clicks link Back to API data sets @@ -78,7 +78,7 @@ Approve first release user clicks link Sign off user approves release for immediate publication -Create a second draft release via api +Create a second draft release user navigates to publication page from dashboard ${PUBLICATION_NAME} user creates release from publication page ${PUBLICATION_NAME} Academic year 3010 @@ -103,19 +103,19 @@ Add data guidance to second release user clicks button Save guidance -Create a different version of an API dataset with major changes +Create a different version of an API data set with major changes user scrolls to the top of the page user clicks link API data sets user waits until h2 is visible API data sets user waits until h3 is visible Current live API data sets - user checks table column heading contains 1 1 Version xpath://table[@data-testid="live-api-data-sets"] + user checks table column heading contains 1 1 Version testid:live-api-data-sets user clicks button in table cell 1 3 Create new version - ... xpath://table[@data-testid="live-api-data-sets"] + ... testid:live-api-data-sets ${modal}= user waits until modal is visible Create a new API data set version - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_2_NAME} + user chooses select option name:releaseFileId ${SUBJECT_2_NAME} user clicks button Confirm new data set version user waits until page finishes loading @@ -170,7 +170,7 @@ User navigates to 'changelog and guidance notes' page and update relevant detail user clicks link by index View changelog and guidance notes 1 user waits until page contains API data set changelog - user enters text into element css:textarea[id="guidanceNotesForm-notes"] + user enters text into element name:notes ... Content for the public guidance notes user clicks button Save public guidance notes @@ -180,7 +180,7 @@ User navigates to 'changelog and guidance notes' page and update relevant detail User clicks on 'View preview token log' link inside the 'Draft version details' section user clicks link by index View changelog and guidance notes 2 -Validate the contents in the 'API dataset changelog' page. +Validate the contents in the 'API data set changelog' page. user waits until page contains API data set changelog user waits until page contains Content for the public guidance notes @@ -201,8 +201,8 @@ Verify newly published release is on Find Statistics page User navigates to data catalogue page user navigates to data catalogue page on public frontend -Search with 2nd API dataset - user clicks element id:searchForm-search +Search for the new data set version + user clicks element name:search user presses keys ${PUBLICATION_NAME} user clicks radio API data sets only @@ -210,17 +210,17 @@ Search with 2nd API dataset user clicks radio Newest user checks summary list contains Status This is the latest data - ... parent=testid:data-set-file-summary-${SUBJECT_2_NAME} + ... testid:data-set-file-summary-${SUBJECT_2_NAME} user checks summary list contains Status Available by API - ... parent=testid:data-set-file-summary-${SUBJECT_2_NAME} + ... testid:data-set-file-summary-${SUBJECT_2_NAME} user checks page contains link ${SUBJECT_2_NAME} -User clicks on 2nd API dataset link +User clicks on the new data set version link user clicks link ${SUBJECT_2_NAME} user waits until page finishes loading user waits until h1 is visible ${SUBJECT_2_NAME} -User checks relevant headings exist on API dataset details page +User checks relevant headings exist on API data set details page user waits until h2 is visible Data set details user waits until h2 is visible Data set preview user waits until h2 is visible Variables in this data set @@ -236,7 +236,7 @@ User verifies major changes in the 'API data set changelog' section user waits until h3 is visible Major changes for version 2.0 ${major_changes_Section}= get child element id:apiChangelog testid:major-changes - ${school_types_filter}= user checks changelog section contains deleted filter ${major_changes_Section} + ${school_types_filter}= user checks changelog section contains deleted filter ${major_changes_section} ... School type user checks page does not contain element testid:minor-changes diff --git a/tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot b/tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot index 6ab31eb3756..75d4c186f76 100644 --- a/tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot +++ b/tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot @@ -51,22 +51,22 @@ Add data guidance to subjects user clicks button Save guidance -Create 1st API dataset +Create the initial API data set version user scrolls to the top of the page user clicks link API data sets user waits until h2 is visible API data sets user clicks button Create API data set ${modal}= user waits until modal is visible Create a new API data set - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_1_NAME} + user chooses select option name:releaseFileId ${SUBJECT_1_NAME} user clicks button Confirm new API data set user waits until page finishes loading user waits until modal is not visible Create a new API data set -User waits until the 1st API dataset status changes to 'Ready' +User waits until the initial API data set version's status changes to "Ready" user waits until h3 is visible Draft version details - wait until keyword succeeds 10x %{WAIT_SMALL}s Verify status of API Datasets Ready + wait until keyword succeeds 10x %{WAIT_SMALL}s Verify status of API data sets Ready Add headline text block to Content page user clicks link Back to API data sets @@ -78,7 +78,7 @@ Approve first release user clicks link Sign off user approves release for immediate publication -Create a second draft release via api +Create a second draft release user navigates to publication page from dashboard ${PUBLICATION_NAME} user creates release from publication page ${PUBLICATION_NAME} Academic year 3010 @@ -105,7 +105,7 @@ Add data guidance to second release user clicks button Save guidance -Create a different version of an API dataset with minor changes +Create a new version of the API data set with minor changes user scrolls to the top of the page user clicks link API data sets user waits until h2 is visible API data sets @@ -117,7 +117,7 @@ Create a different version of an API dataset with minor changes ... testid:live-api-data-sets ${modal}= user waits until modal is visible Create a new API data set version - user chooses select option id:apiDataSetCreateForm-releaseFileId ${SUBJECT_2_NAME} + user chooses select option name:releaseFileId ${SUBJECT_2_NAME} user clicks button Confirm new data set version user waits until page finishes loading @@ -271,7 +271,7 @@ User navigates to 'changelog and guidance notes' page and update relevant detail user clicks link by index View changelog and guidance notes 1 user waits until page contains API data set changelog - user enters text into element css:textarea[id="guidanceNotesForm-notes"] + user enters text into element name:notes ... Content for the public guidance notes user clicks button Save public guidance notes @@ -281,7 +281,7 @@ User navigates to 'changelog and guidance notes' page and update relevant detail User clicks on 'View preview token log' link inside the 'Draft version details' section user clicks link by index View changelog and guidance notes 2 -Validate the contents in the 'API dataset changelog' page. +Validate the contents in the 'API data set changelog' page. user waits until page contains API data set changelog user waits until page contains Content for the public guidance notes @@ -302,8 +302,8 @@ Verify newly published release is on Find Statistics page User navigates to data catalogue page user navigates to data catalogue page on public frontend -Search with 2nd API dataset - user clicks element id:searchForm-search +Search for the new API data set version + user clicks element name:search user presses keys ${PUBLICATION_NAME} user clicks radio API data sets only @@ -311,17 +311,17 @@ Search with 2nd API dataset user clicks radio Newest user checks summary list contains Status This is the latest data - ... parent=testid:data-set-file-summary-${SUBJECT_2_NAME} + ... testid:data-set-file-summary-${SUBJECT_2_NAME} user checks summary list contains Status Available by API - ... parent=testid:data-set-file-summary-${SUBJECT_2_NAME} + ... testid:data-set-file-summary-${SUBJECT_2_NAME} user checks page contains link ${SUBJECT_2_NAME} -User clicks on 2nd API dataset link +User clicks on the new data set version link user clicks link ${SUBJECT_2_NAME} user waits until page finishes loading user waits until h1 is visible ${SUBJECT_2_NAME} -User checks relevant headings exist on API dataset details page +User checks relevant headings exist on the data set details page user waits until h2 is visible Data set details user waits until h2 is visible Data set preview user waits until h2 is visible Variables in this data set @@ -340,15 +340,15 @@ User verifies minor changes in the 'API data set changelog' section ${school_types_filter}= user checks changelog section contains updated filter ${minor_changes_section} ... School type ${updated_school_types_total}= user checks changed facet contains option ${school_types_filter} Total - user checks changed option contains description ${updated_school_types_total} + user checks element contains ${updated_school_types_total} ... label changed to: State-funded primary and secondary - user checks changed option contains description ${updated_school_types_total} no longer an aggregate + user checks element contains ${updated_school_types_total} no longer an aggregate ${updated_regional_options}= user checks changelog section contains updated location level ... ${minor_changes_section} Regional ${updated_yorkshire_and_humber}= user checks changed facet contains option ${updated_regional_options} ... Yorkshire and The Humber - user checks changed option contains description ${updated_yorkshire_and_humber} label changed to: Yorkshire + user checks element contains ${updated_yorkshire_and_humber} label changed to: Yorkshire User verifies no major changes are present user checks page does not contain element testid:major-changes @@ -378,9 +378,3 @@ user checks changed facet contains option ${option}= user checks element contains child element ${changed_facet} ... xpath://li[@data-testid="updated-item" and contains(., "${option_label}")] RETURN ${option} - -user checks changed option contains description - [Arguments] - ... ${changed_option} - ... ${description} - user checks element contains ${changed_option} ${description} From d6934a4f663499b2572e9c8b775ad4894251e12c Mon Sep 17 00:00:00 2001 From: Mark Youngman Date: Tue, 15 Oct 2024 11:04:54 +0100 Subject: [PATCH 78/80] EES-5573 Hide delete user button --- .../src/pages/bau/BauUsersPage.tsx | 35 +++++++++---------- .../pages/bau/__tests__/BauUsersPage.test.tsx | 21 ++++++----- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/src/explore-education-statistics-admin/src/pages/bau/BauUsersPage.tsx b/src/explore-education-statistics-admin/src/pages/bau/BauUsersPage.tsx index 6a0551f41ef..ed4bfb5d03a 100644 --- a/src/explore-education-statistics-admin/src/pages/bau/BauUsersPage.tsx +++ b/src/explore-education-statistics-admin/src/pages/bau/BauUsersPage.tsx @@ -1,26 +1,24 @@ import Link from '@admin/components/Link'; import Page from '@admin/components/Page'; import userService from '@admin/services/userService'; -import ButtonText from '@common/components/ButtonText'; import LoadingSpinner from '@common/components/LoadingSpinner'; import useAsyncRetry from '@common/hooks/useAsyncRetry'; -import logger from '@common/services/logger'; import React from 'react'; import styles from './BauUsersPage.module.scss'; const BauUsersPage = () => { const { value, isLoading } = useAsyncRetry(() => userService.getUsers()); - const handleDeleteUser = async (userEmail: string) => { - await userService - .deleteUser(userEmail) - .then(() => { - window.location.reload(); - }) - .catch(error => { - logger.info(`Error encountered when deleting the user - ${error}`); - }); - }; + // const handleDeleteUser = async (userEmail: string) => { // EES-5573 + // await userService + // .deleteUser(userEmail) + // .then(() => { + // window.location.reload(); + // }) + // .catch(error => { + // logger.info(`Error encountered when deleting the user - ${error}`); + // }); + // }; return ( { > Manage - handleDeleteUser(user.email)} - className={styles.deleteUserButton} - > - Delete - + {/* EES-5573 */} + {/* handleDeleteUser(user.email)} */} + {/* className={styles.deleteUserButton} */} + {/* > */} + {/* Delete */} + {/* */} ))} diff --git a/src/explore-education-statistics-admin/src/pages/bau/__tests__/BauUsersPage.test.tsx b/src/explore-education-statistics-admin/src/pages/bau/__tests__/BauUsersPage.test.tsx index 4249d7ace84..cb09eb84daf 100644 --- a/src/explore-education-statistics-admin/src/pages/bau/__tests__/BauUsersPage.test.tsx +++ b/src/explore-education-statistics-admin/src/pages/bau/__tests__/BauUsersPage.test.tsx @@ -4,8 +4,7 @@ import _userService, { } from '@admin/services/userService'; import { MemoryRouter } from 'react-router'; import { TestConfigContextProvider } from '@admin/contexts/ConfigContext'; -import { render, screen, waitFor } from '@testing-library/react'; -import userEvent from '@testing-library/user-event'; +import { render } from '@testing-library/react'; import BauUsersPage from '../BauUsersPage'; jest.mock('@admin/services/userService'); @@ -31,9 +30,9 @@ describe('BauUsersPage', () => { renderPage(); - await waitFor(() => { - expect(screen.getByText('Delete')).toBeInTheDocument(); - }); + // await waitFor(() => { // EES-5573 + // expect(screen.getByText('Delete')).toBeInTheDocument(); + // }); }); test('calls user service when delete user button is clicked', async () => { @@ -42,12 +41,12 @@ describe('BauUsersPage', () => { renderPage(); - await waitFor(() => { - expect(screen.getByText('Delete')).toBeInTheDocument(); - }); - await userEvent.click(screen.getByRole('button', { name: 'Delete' })); - - expect(userService.deleteUser).toHaveBeenCalled(); + // await waitFor(() => { // EES-5573 + // expect(screen.getByText('Delete')).toBeInTheDocument(); + // }); + // await userEvent.click(screen.getByRole('button', { name: 'Delete' })); + // + // expect(userService.deleteUser).toHaveBeenCalled(); }); function renderPage() { From b7e10585c67be617d6cb349cf213cd63f7598053 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Fri, 11 Oct 2024 11:12:29 +0100 Subject: [PATCH 79/80] EES-5492 - broke orchestrations into their own simplified files and made static, to avoid issues whereby exceptions thrown during the construction of Function classes caused the ImportMetadata stage to lock up --- ...teNextDataSetVersionImportFunctionTests.cs | 4 +- .../Functions/CreateDataSetFunctionTests.cs | 6 +- ...NextDataSetVersionMappingsFunctionTests.cs | 3 +- ...OfNextDataSetVersionImportFunctionTests.cs | 98 +--------------- ...tDataSetVersionImportOrchestrationTests.cs | 105 ++++++++++++++++++ ...ocessInitialDataSetVersionFunctionTests.cs | 104 +---------------- ...InitialDataSetVersionOrchestrationTests.cs | 103 +++++++++++++++++ ...NextDataSetVersionMappingsFunctionTests.cs | 95 +--------------- ...ataSetVersionMappingsOrchestrationTests.cs | 92 +++++++++++++++ .../ProcessorFunctionsIntegrationTest.cs | 6 +- .../Functions/ActivityNames.cs | 14 +-- ...InitialDataSetVersionProcessingFunction.cs | 30 +++++ ...ompleteNextDataSetVersionImportFunction.cs | 2 +- .../Functions/CreateDataSetFunction.cs | 3 +- ...reateNextDataSetVersionMappingsFunction.cs | 2 +- ...ompletionOfNextDataSetVersionFunctions.cs} | 38 +------ ...letionOfNextDataSetVersionOrchestration.cs | 44 ++++++++ ...cessInitialDataSetVersionOrchestration.cs} | 28 +---- ...essNextDataSetVersionMappingsFunctions.cs} | 36 +----- ...NextDataSetVersionMappingsOrchestration.cs | 40 +++++++ 20 files changed, 447 insertions(+), 406 deletions(-) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionOrchestrationTests.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteInitialDataSetVersionProcessingFunction.cs rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/{ProcessCompletionOfNextDataSetVersionFunction.cs => ProcessCompletionOfNextDataSetVersionFunctions.cs} (64%) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionOrchestration.cs rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/{ProcessInitialDataSetVersionFunction.cs => ProcessInitialDataSetVersionOrchestration.cs} (57%) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/{ProcessNextDataSetVersionMappingsFunction.cs => ProcessNextDataSetVersionMappingsFunctions.cs} (56%) create mode 100644 src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsOrchestration.cs diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteNextDataSetVersionImportFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteNextDataSetVersionImportFunctionTests.cs index 25b2a937bf1..a51130b16ac 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteNextDataSetVersionImportFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteNextDataSetVersionImportFunctionTests.cs @@ -48,8 +48,8 @@ public async Task Success() StartOrchestrationOptions? startOrchestrationOptions = null; durableTaskClientMock.Setup(client => client.ScheduleNewOrchestrationInstanceAsync( - nameof(ProcessCompletionOfNextDataSetVersionFunction - .ProcessCompletionOfNextDataSetVersion), + nameof(ProcessCompletionOfNextDataSetVersionOrchestration + .ProcessCompletionOfNextDataSetVersionImport), It.IsAny(), It.IsAny(), It.IsAny())) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateDataSetFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateDataSetFunctionTests.cs index b6d7e00d5a9..697206ad12b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateDataSetFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateDataSetFunctionTests.cs @@ -58,7 +58,7 @@ await AddTestData(context => StartOrchestrationOptions? startOrchestrationOptions = null; durableTaskClientMock.Setup(client => client.ScheduleNewOrchestrationInstanceAsync( - nameof(ProcessInitialDataSetVersionFunction.ProcessInitialDataSetVersion), + nameof(ProcessInitialDataSetVersionOrchestration.ProcessInitialDataSetVersion), It.IsAny(), It.IsAny(), It.IsAny())) @@ -163,8 +163,8 @@ public async Task ReleaseFileIdHasDataSetVersion_ReturnsValidationProblem() DataSet dataSet = DataFixture.DefaultDataSet(); DataSetVersion dataSetVersion = DataFixture.DefaultDataSetVersion() - .WithRelease(DataFixture.DefaultDataSetVersionRelease() - .WithReleaseFileId(releaseFile.Id)) + .WithRelease(DataFixture.DefaultDataSetVersionRelease() + .WithReleaseFileId(releaseFile.Id)) .WithDataSet(dataSet); await AddTestData(context => diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionMappingsFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionMappingsFunctionTests.cs index 0cb063daf22..24c47a0b78e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionMappingsFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CreateNextDataSetVersionMappingsFunctionTests.cs @@ -41,7 +41,8 @@ public async Task Success() StartOrchestrationOptions? startOrchestrationOptions = null; durableTaskClientMock.Setup(client => client.ScheduleNewOrchestrationInstanceAsync( - nameof(ProcessNextDataSetVersionMappingsFunction.ProcessNextDataSetVersionMappings), + nameof(ProcessNextDataSetVersionMappingsFunctionOrchestration + .ProcessNextDataSetVersionMappings), It.IsAny(), It.IsAny(), It.IsAny())) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs index 382b84d6d4b..466bc9c2dc4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs @@ -7,14 +7,9 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Utils; -using Microsoft.DurableTask; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging.Abstractions; -using Moq; -using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; using FilterMeta = GovUk.Education.ExploreEducationStatistics.Public.Data.Model.FilterMeta; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; @@ -37,93 +32,6 @@ public abstract class ProcessCompletionOfNextDataSetVersionImportFunctionTests( TimePeriodsTable.ParquetFile ]; - public class ProcessCompletionOfNextDataSetVersionImportTests( - ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessCompletionOfNextDataSetVersionImportFunctionTests(fixture) - { - [Fact] - public async Task Success() - { - var mockOrchestrationContext = DefaultMockOrchestrationContext(); - var activitySequence = new MockSequence(); - - string[] expectedActivitySequence = - [ - ActivityNames.UpdateFileStoragePath, - ActivityNames.ImportMetadata, - ActivityNames.CreateChanges, - ActivityNames.ImportData, - ActivityNames.WriteDataFiles, - ActivityNames.CompleteNextDataSetVersionImportProcessing - ]; - - foreach (var activityName in expectedActivitySequence) - { - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => context.CallActivityAsync(activityName, - mockOrchestrationContext.Object.InstanceId, - null)) - .Returns(Task.CompletedTask); - } - - await ProcessCompletionOfNextDataSetVersionImport(mockOrchestrationContext.Object); - - VerifyAllMocks(mockOrchestrationContext); - } - - [Fact] - public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() - { - var mockOrchestrationContext = DefaultMockOrchestrationContext(); - - var activitySequence = new MockSequence(); - - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync(ActivityNames.UpdateFileStoragePath, - mockOrchestrationContext.Object.InstanceId, - null)) - .Throws(); - - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync(ActivityNames.HandleProcessingFailure, - mockOrchestrationContext.Object.InstanceId, - null)) - .Returns(Task.CompletedTask); - - await ProcessCompletionOfNextDataSetVersionImport(mockOrchestrationContext.Object); - - VerifyAllMocks(mockOrchestrationContext); - } - - private async Task ProcessCompletionOfNextDataSetVersionImport(TaskOrchestrationContext orchestrationContext) - { - var function = GetRequiredService(); - await function.ProcessCompletionOfNextDataSetVersion( - orchestrationContext, - new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); - } - - private static Mock DefaultMockOrchestrationContext(Guid? instanceId = null) - { - var mock = new Mock(MockBehavior.Strict); - - mock.Setup(context => - context.CreateReplaySafeLogger( - nameof(ProcessCompletionOfNextDataSetVersionFunction.ProcessCompletionOfNextDataSetVersion))) - .Returns(NullLogger.Instance); - - mock.SetupGet(context => context.InstanceId) - .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); - - return mock; - } - } - public abstract class CreateChangesTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessCompletionOfNextDataSetVersionImportFunctionTests(fixture) @@ -132,7 +40,7 @@ public abstract class CreateChangesTests( protected async Task CreateChanges(Guid instanceId) { - var function = GetRequiredService(); + var function = GetRequiredService(); await function.CreateChanges(instanceId, CancellationToken.None); } } @@ -3132,7 +3040,7 @@ public async Task Success_PathNotUpdated() private async Task UpdateFileStoragePath(Guid instanceId) { - var function = GetRequiredService(); + var function = GetRequiredService(); await function.UpdateFileStoragePath(instanceId, CancellationToken.None); } } @@ -3190,7 +3098,7 @@ public async Task DuckDbFileIsDeleted() private async Task CompleteProcessing(Guid instanceId) { - var function = GetRequiredService(); + var function = GetRequiredService(); await function.CompleteNextDataSetVersionImportProcessing(instanceId, CancellationToken.None); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs new file mode 100644 index 00000000000..e2250fb24e1 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs @@ -0,0 +1,105 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Extensions; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; + +public abstract class ProcessCompletionOfNextDataSetVersionImportOrchestrationTests +{ + public class ProcessCompletionOfNextDataSetVersionImportTests + { + [Fact] + public async Task Success() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + var activitySequence = new MockSequence(); + + // Expect an entity lock to be acquired for calling the ImportMetadata activity + var mockEntityFeature = new Mock(MockBehavior.Strict); + mockEntityFeature.SetupLockForActivity(ActivityNames.ImportMetadata); + mockOrchestrationContext.SetupGet(context => context.Entities) + .Returns(mockEntityFeature.Object); + + string[] expectedActivitySequence = + [ + ActivityNames.UpdateFileStoragePath, + ActivityNames.ImportMetadata, + ActivityNames.CreateChanges, + ActivityNames.ImportData, + ActivityNames.WriteDataFiles, + ActivityNames.CompleteNextDataSetVersionImportProcessing + ]; + + foreach (var activityName in expectedActivitySequence) + { + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => context.CallActivityAsync(activityName, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + } + + await ProcessCompletionOfNextDataSetVersionImport(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext, mockEntityFeature); + } + + [Fact] + public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + + var activitySequence = new MockSequence(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.UpdateFileStoragePath, + mockOrchestrationContext.Object.InstanceId, + null)) + .Throws(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.HandleProcessingFailure, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + + await ProcessCompletionOfNextDataSetVersionImport(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext); + } + + private static async Task ProcessCompletionOfNextDataSetVersionImport( + TaskOrchestrationContext orchestrationContext) + { + await ProcessCompletionOfNextDataSetVersionOrchestration.ProcessCompletionOfNextDataSetVersionImport( + orchestrationContext, + new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); + } + + private static Mock DefaultMockOrchestrationContext(Guid? instanceId = null) + { + var mock = new Mock(MockBehavior.Strict); + + mock.Setup(context => + context.CreateReplaySafeLogger( + nameof(ProcessCompletionOfNextDataSetVersionOrchestration + .ProcessCompletionOfNextDataSetVersionImport))) + .Returns(NullLogger.Instance); + + mock.SetupGet(context => context.InstanceId) + .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); + + return mock; + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs index b42bea901ab..ce333b07ee1 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs @@ -3,15 +3,8 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Parquet.Tables; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Extensions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; -using Microsoft.DurableTask; -using Microsoft.DurableTask.Entities; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging.Abstractions; -using Moq; -using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; @@ -33,99 +26,6 @@ public abstract class ProcessInitialDataSetVersionFunctionTests( TimePeriodsTable.ParquetFile ]; - public class ProcessInitialDataSetVersionTests( - ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessInitialDataSetVersionFunctionTests(fixture) - { - [Fact] - public async Task Success() - { - var mockOrchestrationContext = DefaultMockOrchestrationContext(); - - // Expect an entity lock to be acquired for calling the ImportMetadata activity - var mockEntityFeature = new Mock(MockBehavior.Strict); - mockEntityFeature.SetupLockForActivity(ActivityNames.ImportMetadata); - mockOrchestrationContext.SetupGet(context => context.Entities) - .Returns(mockEntityFeature.Object); - - var activitySequence = new MockSequence(); - - string[] expectedActivitySequence = - [ - ActivityNames.CopyCsvFiles, - ActivityNames.ImportMetadata, - ActivityNames.ImportData, - ActivityNames.WriteDataFiles, - ActivityNames.CompleteInitialDataSetVersionProcessing - ]; - - foreach (var activityName in expectedActivitySequence) - { - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => context.CallActivityAsync(activityName, - mockOrchestrationContext.Object.InstanceId, - null)) - .Returns(Task.CompletedTask); - } - - await ProcessInitialDataSetVersion(mockOrchestrationContext.Object); - - VerifyAllMocks(mockOrchestrationContext, mockEntityFeature); - } - - [Fact] - public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() - { - var mockOrchestrationContext = DefaultMockOrchestrationContext(); - - var activitySequence = new MockSequence(); - - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync(ActivityNames.CopyCsvFiles, - mockOrchestrationContext.Object.InstanceId, - null)) - .Throws(); - - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync(ActivityNames.HandleProcessingFailure, - mockOrchestrationContext.Object.InstanceId, - null)) - .Returns(Task.CompletedTask); - - await ProcessInitialDataSetVersion(mockOrchestrationContext.Object); - - VerifyAllMocks(mockOrchestrationContext); - } - - private async Task ProcessInitialDataSetVersion(TaskOrchestrationContext orchestrationContext) - { - var function = GetRequiredService(); - await function.ProcessInitialDataSetVersion( - orchestrationContext, - new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); - } - - private static Mock DefaultMockOrchestrationContext(Guid? instanceId = null) - { - var mock = new Mock(); - - mock.Setup(context => - context.CreateReplaySafeLogger( - nameof(ProcessInitialDataSetVersionFunction.ProcessInitialDataSetVersion))) - .Returns(NullLogger.Instance); - - mock.SetupGet(context => context.InstanceId) - .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); - - return mock; - } - } - public class CompleteInitialDataSetVersionProcessingTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessInitialDataSetVersionFunctionTests(fixture) @@ -179,8 +79,8 @@ public async Task DuckDbFileIsDeleted() private async Task CompleteProcessing(Guid instanceId) { - var function = GetRequiredService(); - await function.CompleteNextDataSetVersionImportProcessing(instanceId, CancellationToken.None); + var function = GetRequiredService(); + await function.CompleteInitialDataSetVersionProcessing(instanceId, CancellationToken.None); } } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionOrchestrationTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionOrchestrationTests.cs new file mode 100644 index 00000000000..97f1e35de16 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionOrchestrationTests.cs @@ -0,0 +1,103 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Extensions; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; + +public abstract class ProcessInitialDataSetVersionOrchestrationTests +{ + public class ProcessInitialDataSetVersionTests + { + [Fact] + public async Task Success() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + + // Expect an entity lock to be acquired for calling the ImportMetadata activity + var mockEntityFeature = new Mock(MockBehavior.Strict); + mockEntityFeature.SetupLockForActivity(ActivityNames.ImportMetadata); + mockOrchestrationContext.SetupGet(context => context.Entities) + .Returns(mockEntityFeature.Object); + + var activitySequence = new MockSequence(); + + string[] expectedActivitySequence = + [ + ActivityNames.CopyCsvFiles, + ActivityNames.ImportMetadata, + ActivityNames.ImportData, + ActivityNames.WriteDataFiles, + ActivityNames.CompleteInitialDataSetVersionProcessing + ]; + + foreach (var activityName in expectedActivitySequence) + { + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => context.CallActivityAsync(activityName, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + } + + await ProcessInitialDataSetVersion(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext, mockEntityFeature); + } + + [Fact] + public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + + var activitySequence = new MockSequence(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.CopyCsvFiles, + mockOrchestrationContext.Object.InstanceId, + null)) + .Throws(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.HandleProcessingFailure, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + + await ProcessInitialDataSetVersion(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext); + } + + private static async Task ProcessInitialDataSetVersion(TaskOrchestrationContext orchestrationContext) + { + await ProcessInitialDataSetVersionOrchestration.ProcessInitialDataSetVersion( + orchestrationContext, + new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); + } + + private static Mock DefaultMockOrchestrationContext(Guid? instanceId = null) + { + var mock = new Mock(); + + mock.Setup(context => + context.CreateReplaySafeLogger( + nameof(ProcessInitialDataSetVersionOrchestration.ProcessInitialDataSetVersion))) + .Returns(NullLogger.Instance); + + mock.SetupGet(context => context.InstanceId) + .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); + + return mock; + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs index 77d1b1f78b4..688ac38c48b 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs @@ -9,13 +9,8 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Tests.Fixtures; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Utils; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; -using Microsoft.DurableTask; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging.Abstractions; -using Moq; -using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; @@ -23,90 +18,6 @@ public abstract class ProcessNextDataSetVersionMappingsFunctionTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessorFunctionsIntegrationTest(fixture) { - public class ProcessNextDataSetVersionMappingsTests( - ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessNextDataSetVersionMappingsFunctionTests(fixture) - { - [Fact] - public async Task Success() - { - var mockOrchestrationContext = DefaultMockOrchestrationContext(); - var activitySequence = new MockSequence(); - - string[] expectedActivitySequence = - [ - ActivityNames.CopyCsvFiles, - ActivityNames.CreateMappings, - ActivityNames.ApplyAutoMappings, - ActivityNames.CompleteNextDataSetVersionMappingProcessing, - ]; - - foreach (var activityName in expectedActivitySequence) - { - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => context.CallActivityAsync(activityName, - mockOrchestrationContext.Object.InstanceId, - null)) - .Returns(Task.CompletedTask); - } - - await ProcessNextDataSetVersion(mockOrchestrationContext.Object); - - VerifyAllMocks(mockOrchestrationContext); - } - - [Fact] - public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() - { - var mockOrchestrationContext = DefaultMockOrchestrationContext(); - - var activitySequence = new MockSequence(); - - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync(ActivityNames.CopyCsvFiles, - mockOrchestrationContext.Object.InstanceId, - null)) - .Throws(); - - mockOrchestrationContext - .InSequence(activitySequence) - .Setup(context => - context.CallActivityAsync(ActivityNames.HandleProcessingFailure, - mockOrchestrationContext.Object.InstanceId, - null)) - .Returns(Task.CompletedTask); - - await ProcessNextDataSetVersion(mockOrchestrationContext.Object); - - VerifyAllMocks(mockOrchestrationContext); - } - - private async Task ProcessNextDataSetVersion(TaskOrchestrationContext orchestrationContext) - { - var function = GetRequiredService(); - await function.ProcessNextDataSetVersionMappings( - orchestrationContext, - new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); - } - - private static Mock DefaultMockOrchestrationContext(Guid? instanceId = null) - { - var mock = new Mock(MockBehavior.Strict); - - mock.Setup(context => context.CreateReplaySafeLogger( - nameof(ProcessNextDataSetVersionMappingsFunction.ProcessNextDataSetVersionMappings))) - .Returns(NullLogger.Instance); - - mock.SetupGet(context => context.InstanceId) - .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); - - return mock; - } - } - public abstract class CreateMappingsTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessNextDataSetVersionMappingsFunctionTests(fixture) @@ -115,7 +26,7 @@ public abstract class CreateMappingsTests( protected async Task CreateMappings(Guid instanceId) { - var function = GetRequiredService(); + var function = GetRequiredService(); await function.CreateMappings(instanceId, CancellationToken.None); } } @@ -629,7 +540,7 @@ public abstract class ApplyAutoMappingsTests( protected async Task ApplyAutoMappings(Guid instanceId) { - var function = GetRequiredService(); + var function = GetRequiredService(); await function.ApplyAutoMappings(instanceId, CancellationToken.None); } } @@ -1790,7 +1701,7 @@ public async Task Success() private async Task CompleteProcessing(Guid instanceId) { - var function = GetRequiredService(); + var function = GetRequiredService(); await function.CompleteNextDataSetVersionMappingProcessing(instanceId, CancellationToken.None); } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs new file mode 100644 index 00000000000..252deb849d0 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs @@ -0,0 +1,92 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using Microsoft.DurableTask; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using static GovUk.Education.ExploreEducationStatistics.Common.Tests.Utils.MockUtils; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; + +public abstract class ProcessNextDataSetVersionMappingsOrchestrationTests +{ + public class ProcessNextDataSetVersionMappingsTests + { + [Fact] + public async Task Success() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + var activitySequence = new MockSequence(); + + string[] expectedActivitySequence = + [ + ActivityNames.CopyCsvFiles, + ActivityNames.CreateMappings, + ActivityNames.ApplyAutoMappings, + ActivityNames.CompleteNextDataSetVersionMappingProcessing, + ]; + + foreach (var activityName in expectedActivitySequence) + { + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => context.CallActivityAsync(activityName, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + } + + await ProcessNextDataSetVersion(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext); + } + + [Fact] + public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() + { + var mockOrchestrationContext = DefaultMockOrchestrationContext(); + + var activitySequence = new MockSequence(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.CopyCsvFiles, + mockOrchestrationContext.Object.InstanceId, + null)) + .Throws(); + + mockOrchestrationContext + .InSequence(activitySequence) + .Setup(context => + context.CallActivityAsync(ActivityNames.HandleProcessingFailure, + mockOrchestrationContext.Object.InstanceId, + null)) + .Returns(Task.CompletedTask); + + await ProcessNextDataSetVersion(mockOrchestrationContext.Object); + + VerifyAllMocks(mockOrchestrationContext); + } + + private async Task ProcessNextDataSetVersion(TaskOrchestrationContext orchestrationContext) + { + await ProcessNextDataSetVersionMappingsFunctionOrchestration.ProcessNextDataSetVersionMappings( + orchestrationContext, + new ProcessDataSetVersionContext { DataSetVersionId = Guid.NewGuid() }); + } + + private static Mock DefaultMockOrchestrationContext(Guid? instanceId = null) + { + var mock = new Mock(MockBehavior.Strict); + + mock.Setup(context => context.CreateReplaySafeLogger( + nameof(ProcessNextDataSetVersionMappingsFunctionOrchestration.ProcessNextDataSetVersionMappings))) + .Returns(NullLogger.Instance); + + mock.SetupGet(context => context.InstanceId) + .Returns(instanceId?.ToString() ?? Guid.NewGuid().ToString()); + + return mock; + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs index e9d51bad276..b15821ee4c5 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/ProcessorFunctionsIntegrationTest.cs @@ -288,11 +288,11 @@ protected override IEnumerable GetFunctionTypes() return [ typeof(CreateDataSetFunction), - typeof(ProcessInitialDataSetVersionFunction), + typeof(CompleteInitialDataSetVersionProcessingFunction), typeof(CreateNextDataSetVersionMappingsFunction), - typeof(ProcessNextDataSetVersionMappingsFunction), + typeof(ProcessNextDataSetVersionMappingsFunctions), typeof(CompleteNextDataSetVersionImportFunction), - typeof(ProcessCompletionOfNextDataSetVersionFunction), + typeof(ProcessCompletionOfNextDataSetVersionFunctions), typeof(DeleteDataSetVersionFunction), typeof(CopyCsvFilesFunction), typeof(ImportMetadataFunction), diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs index b990d7e2b45..f1f002e15c4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ActivityNames.cs @@ -9,17 +9,17 @@ internal static class ActivityNames public const string HandleProcessingFailure = nameof(HandleProcessingFailureFunction.HandleProcessingFailure); public const string CompleteInitialDataSetVersionProcessing = - nameof(ProcessInitialDataSetVersionFunction.CompleteInitialDataSetVersionProcessing); + nameof(CompleteInitialDataSetVersionProcessingFunction.CompleteInitialDataSetVersionProcessing); - public const string CreateMappings = nameof(ProcessNextDataSetVersionMappingsFunction.CreateMappings); - public const string ApplyAutoMappings = nameof(ProcessNextDataSetVersionMappingsFunction.ApplyAutoMappings); + public const string CreateMappings = nameof(ProcessNextDataSetVersionMappingsFunctions.CreateMappings); + public const string ApplyAutoMappings = nameof(ProcessNextDataSetVersionMappingsFunctions.ApplyAutoMappings); public const string CompleteNextDataSetVersionMappingProcessing = - nameof(ProcessNextDataSetVersionMappingsFunction.CompleteNextDataSetVersionMappingProcessing); + nameof(ProcessNextDataSetVersionMappingsFunctions.CompleteNextDataSetVersionMappingProcessing); public const string CreateChanges = - nameof(ProcessCompletionOfNextDataSetVersionFunction.CreateChanges); + nameof(ProcessCompletionOfNextDataSetVersionFunctions.CreateChanges); public const string UpdateFileStoragePath = - nameof(ProcessCompletionOfNextDataSetVersionFunction.UpdateFileStoragePath); + nameof(ProcessCompletionOfNextDataSetVersionFunctions.UpdateFileStoragePath); public const string CompleteNextDataSetVersionImportProcessing = - nameof(ProcessCompletionOfNextDataSetVersionFunction.CompleteNextDataSetVersionImportProcessing); + nameof(ProcessCompletionOfNextDataSetVersionFunctions.CompleteNextDataSetVersionImportProcessing); } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteInitialDataSetVersionProcessingFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteInitialDataSetVersionProcessingFunction.cs new file mode 100644 index 00000000000..b0d20ac0fbb --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteInitialDataSetVersionProcessingFunction.cs @@ -0,0 +1,30 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; +using Microsoft.Azure.Functions.Worker; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public class CompleteInitialDataSetVersionProcessingFunction( + PublicDataDbContext publicDataDbContext, + IDataSetVersionPathResolver dataSetVersionPathResolver) : BaseProcessDataSetVersionFunction(publicDataDbContext) +{ + [Function(ActivityNames.CompleteInitialDataSetVersionProcessing)] + public async Task CompleteInitialDataSetVersionProcessing( + [ActivityTrigger] Guid instanceId, + CancellationToken cancellationToken) + { + var dataSetVersionImport = await GetDataSetVersionImport(instanceId, cancellationToken); + await UpdateImportStage(dataSetVersionImport, DataSetVersionImportStage.Completing, cancellationToken); + + var dataSetVersion = dataSetVersionImport.DataSetVersion; + + // Delete the DuckDb database file as it is no longer needed + File.Delete(dataSetVersionPathResolver.DuckDbPath(dataSetVersion)); + + dataSetVersion.Status = DataSetVersionStatus.Draft; + + dataSetVersionImport.Completed = DateTimeOffset.UtcNow; + await publicDataDbContext.SaveChangesAsync(cancellationToken); + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteNextDataSetVersionImportFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteNextDataSetVersionImportFunction.cs index 3351d89d3ec..8273ec0ef3e 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteNextDataSetVersionImportFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CompleteNextDataSetVersionImportFunction.cs @@ -62,7 +62,7 @@ private async Task ProcessCompletionOfNextDataSetVersionImport( CancellationToken cancellationToken) { const string orchestratorName = - nameof(ProcessCompletionOfNextDataSetVersionFunction.ProcessCompletionOfNextDataSetVersion); + nameof(ProcessCompletionOfNextDataSetVersionOrchestration.ProcessCompletionOfNextDataSetVersionImport); var input = new ProcessDataSetVersionContext { DataSetVersionId = dataSetVersionId }; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateDataSetFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateDataSetFunction.cs index aed6af84bed..5660fdfab27 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateDataSetFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateDataSetFunction.cs @@ -59,7 +59,8 @@ private async Task ProcessInitialDataSetVersion( Guid instanceId, CancellationToken cancellationToken) { - const string orchestratorName = nameof(ProcessInitialDataSetVersionFunction.ProcessInitialDataSetVersion); + const string orchestratorName = + nameof(ProcessInitialDataSetVersionOrchestration.ProcessInitialDataSetVersion); var input = new ProcessDataSetVersionContext { DataSetVersionId = dataSetVersionId }; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateNextDataSetVersionMappingsFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateNextDataSetVersionMappingsFunction.cs index 81a6b6058c7..f253abffc88 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateNextDataSetVersionMappingsFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/CreateNextDataSetVersionMappingsFunction.cs @@ -64,7 +64,7 @@ private async Task ProcessNextDataSetVersion( CancellationToken cancellationToken) { const string orchestratorName = - nameof(ProcessNextDataSetVersionMappingsFunction.ProcessNextDataSetVersionMappings); + nameof(ProcessNextDataSetVersionMappingsFunctionOrchestration.ProcessNextDataSetVersionMappings); var input = new ProcessDataSetVersionContext { DataSetVersionId = dataSetVersionId }; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunctions.cs similarity index 64% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunction.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunctions.cs index a71de9a718c..e9bae2c8a76 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionFunctions.cs @@ -5,51 +5,15 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using Microsoft.Azure.Functions.Worker; -using Microsoft.DurableTask; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -public class ProcessCompletionOfNextDataSetVersionFunction( +public class ProcessCompletionOfNextDataSetVersionFunctions( PublicDataDbContext publicDataDbContext, IDataSetVersionPathResolver dataSetVersionPathResolver, IDataSetVersionChangeService dataSetVersionChangeService) : BaseProcessDataSetVersionFunction(publicDataDbContext) { - [Function(nameof(ProcessCompletionOfNextDataSetVersion))] - public async Task ProcessCompletionOfNextDataSetVersion( - [OrchestrationTrigger] TaskOrchestrationContext context, - ProcessDataSetVersionContext input) - { - var logger = context.CreateReplaySafeLogger(nameof(ProcessCompletionOfNextDataSetVersion)); - - logger.LogInformation( - "Processing completion of import for next data set version (InstanceId={InstanceId}, " + - "DataSetVersionId={DataSetVersionId})", - context.InstanceId, - input.DataSetVersionId); - - try - { - await context.CallActivity(ActivityNames.UpdateFileStoragePath, logger, context.InstanceId); - await context.CallActivity(ActivityNames.ImportMetadata, logger, context.InstanceId); - await context.CallActivity(ActivityNames.CreateChanges, logger, context.InstanceId); - await context.CallActivity(ActivityNames.ImportData, logger, context.InstanceId); - await context.CallActivity(ActivityNames.WriteDataFiles, logger, context.InstanceId); - await context.CallActivity(ActivityNames.CompleteNextDataSetVersionImportProcessing, logger, - context.InstanceId); - } - catch (Exception e) - { - logger.LogError(e, - "Activity failed with an exception (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", - context.InstanceId, - input.DataSetVersionId); - - await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); - } - } - [Function(ActivityNames.CreateChanges)] public async Task CreateChanges( [ActivityTrigger] Guid instanceId, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionOrchestration.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionOrchestration.cs new file mode 100644 index 00000000000..57db11c7180 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessCompletionOfNextDataSetVersionOrchestration.cs @@ -0,0 +1,44 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using Microsoft.Azure.Functions.Worker; +using Microsoft.DurableTask; +using Microsoft.Extensions.Logging; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public static class ProcessCompletionOfNextDataSetVersionOrchestration +{ + [Function(nameof(ProcessCompletionOfNextDataSetVersionImport))] + public static async Task ProcessCompletionOfNextDataSetVersionImport( + [OrchestrationTrigger] TaskOrchestrationContext context, + ProcessDataSetVersionContext input) + { + var logger = context.CreateReplaySafeLogger(nameof(ProcessCompletionOfNextDataSetVersionImport)); + + logger.LogInformation( + "Processing completion of import for next data set version (InstanceId={InstanceId}, " + + "DataSetVersionId={DataSetVersionId})", + context.InstanceId, + input.DataSetVersionId); + + try + { + await context.CallActivity(ActivityNames.UpdateFileStoragePath, logger, context.InstanceId); + await context.CallActivityExclusively(ActivityNames.ImportMetadata, logger, context.InstanceId); + await context.CallActivity(ActivityNames.CreateChanges, logger, context.InstanceId); + await context.CallActivity(ActivityNames.ImportData, logger, context.InstanceId); + await context.CallActivity(ActivityNames.WriteDataFiles, logger, context.InstanceId); + await context.CallActivity(ActivityNames.CompleteNextDataSetVersionImportProcessing, logger, + context.InstanceId); + } + catch (Exception e) + { + logger.LogError(e, + "Activity failed with an exception (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", + context.InstanceId, + input.DataSetVersionId); + + await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); + } + } +} diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionOrchestration.cs similarity index 57% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionFunction.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionOrchestration.cs index 9ba387dc7e9..f677a68986c 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessInitialDataSetVersionOrchestration.cs @@ -1,20 +1,15 @@ -using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Services.Interfaces; using Microsoft.Azure.Functions.Worker; using Microsoft.DurableTask; using Microsoft.Extensions.Logging; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -public class ProcessInitialDataSetVersionFunction( - PublicDataDbContext publicDataDbContext, - IDataSetVersionPathResolver dataSetVersionPathResolver) : BaseProcessDataSetVersionFunction(publicDataDbContext) +public static class ProcessInitialDataSetVersionOrchestration { [Function(nameof(ProcessInitialDataSetVersion))] - public async Task ProcessInitialDataSetVersion( + public static async Task ProcessInitialDataSetVersion( [OrchestrationTrigger] TaskOrchestrationContext context, ProcessDataSetVersionContext input) { @@ -44,23 +39,4 @@ await context.CallActivity(ActivityNames.CompleteInitialDataSetVersionProcessing await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); } } - - [Function(ActivityNames.CompleteInitialDataSetVersionProcessing)] - public async Task CompleteInitialDataSetVersionProcessing( - [ActivityTrigger] Guid instanceId, - CancellationToken cancellationToken) - { - var dataSetVersionImport = await GetDataSetVersionImport(instanceId, cancellationToken); - await UpdateImportStage(dataSetVersionImport, DataSetVersionImportStage.Completing, cancellationToken); - - var dataSetVersion = dataSetVersionImport.DataSetVersion; - - // Delete the DuckDb database file as it is no longer needed - File.Delete(dataSetVersionPathResolver.DuckDbPath(dataSetVersion)); - - dataSetVersion.Status = DataSetVersionStatus.Draft; - - dataSetVersionImport.Completed = DateTimeOffset.UtcNow; - await publicDataDbContext.SaveChangesAsync(cancellationToken); - } } diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsFunction.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsFunctions.cs similarity index 56% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsFunction.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsFunctions.cs index c1507e6adc3..a24c1189444 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsFunction.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsFunctions.cs @@ -1,48 +1,14 @@ using GovUk.Education.ExploreEducationStatistics.Public.Data.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Model.Database; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; -using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Services.Interfaces; using Microsoft.Azure.Functions.Worker; -using Microsoft.DurableTask; -using Microsoft.Extensions.Logging; namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; -public class ProcessNextDataSetVersionMappingsFunction( +public class ProcessNextDataSetVersionMappingsFunctions( PublicDataDbContext publicDataDbContext, IDataSetVersionMappingService mappingService) : BaseProcessDataSetVersionFunction(publicDataDbContext) { - [Function(nameof(ProcessNextDataSetVersionMappings))] - public async Task ProcessNextDataSetVersionMappings([OrchestrationTrigger] TaskOrchestrationContext context, - ProcessDataSetVersionContext input) - { - var logger = context.CreateReplaySafeLogger(nameof(ProcessNextDataSetVersionMappings)); - - logger.LogInformation( - "Processing next data set version (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", - context.InstanceId, - input.DataSetVersionId); - - try - { - await context.CallActivity(ActivityNames.CopyCsvFiles, logger, context.InstanceId); - await context.CallActivity(ActivityNames.CreateMappings, logger, context.InstanceId); - await context.CallActivity(ActivityNames.ApplyAutoMappings, logger, context.InstanceId); - await context.CallActivity(ActivityNames.CompleteNextDataSetVersionMappingProcessing, logger, - context.InstanceId); - } - catch (Exception e) - { - logger.LogError(e, - "Activity failed with an exception (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", - context.InstanceId, - input.DataSetVersionId); - - await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); - } - } - [Function(ActivityNames.CreateMappings)] public async Task CreateMappings( [ActivityTrigger] Guid instanceId, diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsOrchestration.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsOrchestration.cs new file mode 100644 index 00000000000..c2dc922c9d9 --- /dev/null +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor/Functions/ProcessNextDataSetVersionMappingsOrchestration.cs @@ -0,0 +1,40 @@ +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Extensions; +using GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Model; +using Microsoft.Azure.Functions.Worker; +using Microsoft.DurableTask; +using Microsoft.Extensions.Logging; + +namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Functions; + +public static class ProcessNextDataSetVersionMappingsFunctionOrchestration +{ + [Function(nameof(ProcessNextDataSetVersionMappings))] + public static async Task ProcessNextDataSetVersionMappings([OrchestrationTrigger] TaskOrchestrationContext context, + ProcessDataSetVersionContext input) + { + var logger = context.CreateReplaySafeLogger(nameof(ProcessNextDataSetVersionMappings)); + + logger.LogInformation( + "Processing next data set version (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", + context.InstanceId, + input.DataSetVersionId); + + try + { + await context.CallActivity(ActivityNames.CopyCsvFiles, logger, context.InstanceId); + await context.CallActivity(ActivityNames.CreateMappings, logger, context.InstanceId); + await context.CallActivity(ActivityNames.ApplyAutoMappings, logger, context.InstanceId); + await context.CallActivity(ActivityNames.CompleteNextDataSetVersionMappingProcessing, logger, + context.InstanceId); + } + catch (Exception e) + { + logger.LogError(e, + "Activity failed with an exception (InstanceId={InstanceId}, DataSetVersionId={DataSetVersionId})", + context.InstanceId, + input.DataSetVersionId); + + await context.CallActivity(ActivityNames.HandleProcessingFailure, logger, context.InstanceId); + } + } +} From 93b5d5e585ce252324c1ae480be7e8732a41e9d7 Mon Sep 17 00:00:00 2001 From: Duncan Watson Date: Mon, 30 Sep 2024 09:45:57 +0100 Subject: [PATCH 80/80] EES-5492 - renamed a lot of test classes and method names in response to PR comments! --- ...nitialDataSetVersionProcessingFunctionTests.cs} | 12 ++++++------ ...ompletionOfNextDataSetVersionFunctionsTests.cs} | 14 +++++++------- ...etionOfNextDataSetVersionOrchestrationTests.cs} | 2 +- ...essNextDataSetVersionMappingsFunctionsTests.cs} | 8 ++++---- ...NextDataSetVersionMappingsOrchestrationTests.cs | 6 +++--- .../public_api_minor_manual_changes.robot | 8 ++++---- 6 files changed, 25 insertions(+), 25 deletions(-) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/{ProcessInitialDataSetVersionFunctionTests.cs => CompleteInitialDataSetVersionProcessingFunctionTests.cs} (88%) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/{ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs => ProcessCompletionOfNextDataSetVersionFunctionsTests.cs} (99%) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/{ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs => ProcessCompletionOfNextDataSetVersionOrchestrationTests.cs} (98%) rename src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/{ProcessNextDataSetVersionMappingsFunctionTests.cs => ProcessNextDataSetVersionMappingsFunctionsTests.cs} (99%) diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteInitialDataSetVersionProcessingFunctionTests.cs similarity index 88% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteInitialDataSetVersionProcessingFunctionTests.cs index ce333b07ee1..17ec1689a0f 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessInitialDataSetVersionFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/CompleteInitialDataSetVersionProcessingFunctionTests.cs @@ -8,7 +8,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; -public abstract class ProcessInitialDataSetVersionFunctionTests( +public abstract class CompleteInitialDataSetVersionProcessingFunctionTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessorFunctionsIntegrationTest(fixture) { @@ -26,9 +26,9 @@ public abstract class ProcessInitialDataSetVersionFunctionTests( TimePeriodsTable.ParquetFile ]; - public class CompleteInitialDataSetVersionProcessingTests( + public class CompleteInitialDataSetVersionProcessingProcessingTests( ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessInitialDataSetVersionFunctionTests(fixture) + : CompleteInitialDataSetVersionProcessingFunctionTests(fixture) { private const DataSetVersionImportStage Stage = DataSetVersionImportStage.Completing; @@ -40,7 +40,7 @@ public async Task Success() var dataSetVersionPathResolver = GetRequiredService(); Directory.CreateDirectory(dataSetVersionPathResolver.DirectoryPath(dataSetVersion)); - await CompleteProcessing(instanceId); + await CompleteInitialDataSetVersionProcessing(instanceId); await using var publicDataDbContext = GetDbContext(); @@ -68,7 +68,7 @@ public async Task DuckDbFileIsDeleted() await File.Create(Path.Combine(directoryPath, filename)).DisposeAsync(); } - await CompleteProcessing(instanceId); + await CompleteInitialDataSetVersionProcessing(instanceId); // Ensure the duck db database file is the only file that was deleted AssertDataSetVersionDirectoryContainsOnlyFiles(dataSetVersion, @@ -77,7 +77,7 @@ public async Task DuckDbFileIsDeleted() .ToArray()); } - private async Task CompleteProcessing(Guid instanceId) + private async Task CompleteInitialDataSetVersionProcessing(Guid instanceId) { var function = GetRequiredService(); await function.CompleteInitialDataSetVersionProcessing(instanceId, CancellationToken.None); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionFunctionsTests.cs similarity index 99% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionFunctionsTests.cs index 466bc9c2dc4..ad18ad3daa2 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionFunctionsTests.cs @@ -14,7 +14,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; -public abstract class ProcessCompletionOfNextDataSetVersionImportFunctionTests( +public abstract class ProcessCompletionOfNextDataSetVersionFunctionsTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessorFunctionsIntegrationTest(fixture) { @@ -34,7 +34,7 @@ public abstract class ProcessCompletionOfNextDataSetVersionImportFunctionTests( public abstract class CreateChangesTests( ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessCompletionOfNextDataSetVersionImportFunctionTests(fixture) + : ProcessCompletionOfNextDataSetVersionFunctionsTests(fixture) { protected const DataSetVersionImportStage Stage = DataSetVersionImportStage.CreatingChanges; @@ -2992,7 +2992,7 @@ .. DataFixture.DefaultTimePeriodMeta() public class UpdateFileStoragePathTests( ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessCompletionOfNextDataSetVersionImportFunctionTests(fixture) + : ProcessCompletionOfNextDataSetVersionFunctionsTests(fixture) { private const DataSetVersionImportStage Stage = DataSetVersionImportStage.ManualMapping; @@ -3047,7 +3047,7 @@ private async Task UpdateFileStoragePath(Guid instanceId) public class CompleteNextDataSetVersionImportProcessingTests( ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessCompletionOfNextDataSetVersionImportFunctionTests(fixture) + : ProcessCompletionOfNextDataSetVersionFunctionsTests(fixture) { private const DataSetVersionImportStage Stage = DataSetVersionImportStage.Completing; @@ -3059,7 +3059,7 @@ public async Task Success() var dataSetVersionPathResolver = GetRequiredService(); Directory.CreateDirectory(dataSetVersionPathResolver.DirectoryPath(dataSetVersion)); - await CompleteProcessing(instanceId); + await CompleteNextDataSetVersionImportProcessing(instanceId); await using var publicDataDbContext = GetDbContext(); @@ -3087,7 +3087,7 @@ public async Task DuckDbFileIsDeleted() await File.Create(Path.Combine(directoryPath, filename)).DisposeAsync(); } - await CompleteProcessing(instanceId); + await CompleteNextDataSetVersionImportProcessing(instanceId); // Ensure the duck db database file is the only file that was deleted AssertDataSetVersionDirectoryContainsOnlyFiles(dataSetVersion, @@ -3096,7 +3096,7 @@ public async Task DuckDbFileIsDeleted() .ToArray()); } - private async Task CompleteProcessing(Guid instanceId) + private async Task CompleteNextDataSetVersionImportProcessing(Guid instanceId) { var function = GetRequiredService(); await function.CompleteNextDataSetVersionImportProcessing(instanceId, CancellationToken.None); diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionOrchestrationTests.cs similarity index 98% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionOrchestrationTests.cs index e2250fb24e1..4cf6c417a07 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionImportOrchestrationTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessCompletionOfNextDataSetVersionOrchestrationTests.cs @@ -9,7 +9,7 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; -public abstract class ProcessCompletionOfNextDataSetVersionImportOrchestrationTests +public abstract class ProcessCompletionOfNextDataSetVersionOrchestrationTests { public class ProcessCompletionOfNextDataSetVersionImportTests { diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionsTests.cs similarity index 99% rename from src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs rename to src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionsTests.cs index 688ac38c48b..ad71c0bd3a4 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsFunctionsTests.cs @@ -14,13 +14,13 @@ namespace GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests.Functions; -public abstract class ProcessNextDataSetVersionMappingsFunctionTests( +public abstract class ProcessNextDataSetVersionMappingsFunctionsTests( ProcessorFunctionsIntegrationTestFixture fixture) : ProcessorFunctionsIntegrationTest(fixture) { public abstract class CreateMappingsTests( ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessNextDataSetVersionMappingsFunctionTests(fixture) + : ProcessNextDataSetVersionMappingsFunctionsTests(fixture) { protected const DataSetVersionImportStage Stage = DataSetVersionImportStage.CreatingMappings; @@ -534,7 +534,7 @@ public async Task Success_Candidates() public abstract class ApplyAutoMappingsTests( ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessNextDataSetVersionMappingsFunctionTests(fixture) + : ProcessNextDataSetVersionMappingsFunctionsTests(fixture) { protected const DataSetVersionImportStage Stage = DataSetVersionImportStage.AutoMapping; @@ -1673,7 +1673,7 @@ public async Task Complete_HasDeletedTimePeriods_MajorUpdate() public class CompleteNextDataSetVersionMappingsMappingProcessingTests( ProcessorFunctionsIntegrationTestFixture fixture) - : ProcessNextDataSetVersionMappingsFunctionTests(fixture) + : ProcessNextDataSetVersionMappingsFunctionsTests(fixture) { private const DataSetVersionImportStage Stage = DataSetVersionImportStage.ManualMapping; diff --git a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs index 252deb849d0..a2d17dc1616 100644 --- a/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs +++ b/src/GovUk.Education.ExploreEducationStatistics.Public.Data.Processor.Tests/Functions/ProcessNextDataSetVersionMappingsOrchestrationTests.cs @@ -35,7 +35,7 @@ public async Task Success() .Returns(Task.CompletedTask); } - await ProcessNextDataSetVersion(mockOrchestrationContext.Object); + await ProcessNextDataSetVersionMappings(mockOrchestrationContext.Object); VerifyAllMocks(mockOrchestrationContext); } @@ -63,12 +63,12 @@ public async Task ActivityFunctionThrowsException_CallsHandleFailureActivity() null)) .Returns(Task.CompletedTask); - await ProcessNextDataSetVersion(mockOrchestrationContext.Object); + await ProcessNextDataSetVersionMappings(mockOrchestrationContext.Object); VerifyAllMocks(mockOrchestrationContext); } - private async Task ProcessNextDataSetVersion(TaskOrchestrationContext orchestrationContext) + private async Task ProcessNextDataSetVersionMappings(TaskOrchestrationContext orchestrationContext) { await ProcessNextDataSetVersionMappingsFunctionOrchestration.ProcessNextDataSetVersionMappings( orchestrationContext, diff --git a/tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot b/tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot index 75d4c186f76..6d9e96e4e72 100644 --- a/tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot +++ b/tests/robot-tests/tests/public_api/public_api_minor_manual_changes.robot @@ -193,7 +193,7 @@ Verify location mapping changes user waits until element contains xpath://table[@data-testid='mappable-table-region']/caption//strong[1] ... 1 mapped location %{WAIT_LONG} -Validate the row headings and its contents in the 'Regions' section(after mapping) +Validate the row headings and its contents in the 'Regions' section after mapping user waits until h3 is visible Locations not found in new data set user checks table column heading contains 1 1 Current data set user checks table column heading contains 1 2 New data set @@ -207,12 +207,12 @@ Validate the row headings and its contents in the 'Regions' section(after mappin user clicks link Back -Validate the version status of location task +Validate the version status of location task is now complete user waits until h3 is visible Draft version tasks user waits until parent contains element testid:map-locations-task link:Map locations user waits until parent contains element id:map-locations-task-status text:Complete user waits until parent contains element testid:map-filters-task link:Map filters - user waits until parent contains element id:map-filters-task-status text:Complete + user waits until parent contains element id:map-filters-task-status text:Incomplete User clicks on Map filters link user clicks link Map filters @@ -247,7 +247,7 @@ Verify filter mapping changes user waits until element contains xpath://table[@data-testid='mappable-table-school_type']/caption//strong[1] ... 1 mapped filter option %{WAIT_LONG} -Validate the row headings and its contents in the 'filters options' section(after mapping) +Validate the row headings and its contents in the 'filters options' section after mapping user waits until h3 is visible Filter options not found in new data set user checks table column heading contains 1 1 Current data set user checks table column heading contains 1 2 New data set