From c5f8eb68e738f02c78478dfc339ec57026fbbdf3 Mon Sep 17 00:00:00 2001 From: xchen Date: Wed, 1 May 2024 20:09:31 -0700 Subject: [PATCH 1/6] github action image build --- .github/workflows/templates-build-push-image.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/templates-build-push-image.yml b/.github/workflows/templates-build-push-image.yml index 60a8380f8..bf457cfc8 100644 --- a/.github/workflows/templates-build-push-image.yml +++ b/.github/workflows/templates-build-push-image.yml @@ -1,5 +1,6 @@ +name: Build and Push (Linux) on: - workflow_call: + workflow_dispatch: inputs: image_name: required: true @@ -15,6 +16,8 @@ jobs: linux: name: Build & Push (Linux) runs-on: ubuntu-latest + permissions: + packages: write steps: - name: Checkout Code uses: actions/checkout@v4 @@ -40,7 +43,7 @@ jobs: uses: docker/login-action@v3 with: registry: ghcr.io - username: tomkerkhove + username: locmai password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push preview image @@ -49,5 +52,5 @@ jobs: build-args: VERSION="${{ env.artifact_full_version }}" context: ./src/ file: ./src/${{ inputs.project_name }}/Dockerfile.linux - tags: ${{ env.image_commit_uri }},${{ env.image_latest_uri }} - push: true + tags: ghcr.io/locmai/${{ env.image_commit_uri }},ghcr.io/locmai/${{ env.image_latest_uri }} + push: true \ No newline at end of file From a39642cfb301b459a0020800cb821b048f5e0f07 Mon Sep 17 00:00:00 2001 From: xchen Date: Wed, 1 May 2024 20:27:55 -0700 Subject: [PATCH 2/6] github action image build --- .github/workflows/templates-build-push-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/templates-build-push-image.yml b/.github/workflows/templates-build-push-image.yml index bf457cfc8..16f5d8440 100644 --- a/.github/workflows/templates-build-push-image.yml +++ b/.github/workflows/templates-build-push-image.yml @@ -43,7 +43,7 @@ jobs: uses: docker/login-action@v3 with: registry: ghcr.io - username: locmai + username: hkfgo password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push preview image @@ -52,5 +52,5 @@ jobs: build-args: VERSION="${{ env.artifact_full_version }}" context: ./src/ file: ./src/${{ inputs.project_name }}/Dockerfile.linux - tags: ghcr.io/locmai/${{ env.image_commit_uri }},ghcr.io/locmai/${{ env.image_latest_uri }} + tags: ghcr.io/hkfgo/${{ env.image_commit_uri }},ghcr.io/hkfgo/${{ env.image_latest_uri }} push: true \ No newline at end of file From 1f28529d6bacb4ff355f7bc60c556af4d47ebb37 Mon Sep 17 00:00:00 2001 From: xchen Date: Sun, 2 Jun 2024 18:44:08 -0700 Subject: [PATCH 3/6] add run time flag for azure monitor SDK integration --- .../AzureMonitorClientFactory.cs | 6 +-- src/Promitor.Agents.Scraper/Docs/Open-Api.xml | 3 +- .../Scheduling/ResourcesScrapingJob.cs | 4 +- .../AzureMonitorIntegrationConfiguration.cs | 1 + .../RuntimeConfigurationUnitTest.cs | 37 +++++++++++++++++++ .../Config/RuntimeConfigurationGenerator.cs | 8 +++- 6 files changed, 51 insertions(+), 8 deletions(-) diff --git a/src/Promitor.Agents.Scraper/AzureMonitorClientFactory.cs b/src/Promitor.Agents.Scraper/AzureMonitorClientFactory.cs index d76430497..a561f2db2 100644 --- a/src/Promitor.Agents.Scraper/AzureMonitorClientFactory.cs +++ b/src/Promitor.Agents.Scraper/AzureMonitorClientFactory.cs @@ -31,14 +31,14 @@ public class AzureMonitorClientFactory /// Options for Azure Monitor integration /// Options for Azure Monitor logging /// Factory to create loggers with - /// Whether to use the new Azure.Monitor.Query package for queries - public IAzureMonitorClient CreateIfNotExists(AzureCloud cloud, string tenantId, string subscriptionId, MetricSinkWriter metricSinkWriter, IAzureScrapingSystemMetricsPublisher azureScrapingSystemMetricsPublisher, IMemoryCache resourceMetricDefinitionMemoryCache, IConfiguration configuration, IOptions azureMonitorIntegrationConfiguration, IOptions azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory, bool useAzureMonitorSdk) + public IAzureMonitorClient CreateIfNotExists(AzureCloud cloud, string tenantId, string subscriptionId, MetricSinkWriter metricSinkWriter, IAzureScrapingSystemMetricsPublisher azureScrapingSystemMetricsPublisher, IMemoryCache resourceMetricDefinitionMemoryCache, IConfiguration configuration, IOptions azureMonitorIntegrationConfiguration, IOptions azureMonitorLoggingConfiguration, ILoggerFactory loggerFactory) { if (_azureMonitorClients.TryGetValue(subscriptionId, out var value)) { return value; } - + + var useAzureMonitorSdk = azureMonitorIntegrationConfiguration.Value.UseAzureMonitorSdk; IAzureMonitorClient azureMonitorClient; if (useAzureMonitorSdk) { azureMonitorClient = CreateNewAzureMonitorQueryClient(cloud, tenantId, subscriptionId, metricSinkWriter, azureScrapingSystemMetricsPublisher, resourceMetricDefinitionMemoryCache, configuration, azureMonitorIntegrationConfiguration, azureMonitorLoggingConfiguration, loggerFactory); diff --git a/src/Promitor.Agents.Scraper/Docs/Open-Api.xml b/src/Promitor.Agents.Scraper/Docs/Open-Api.xml index 08f26d6c2..7299b25fa 100644 --- a/src/Promitor.Agents.Scraper/Docs/Open-Api.xml +++ b/src/Promitor.Agents.Scraper/Docs/Open-Api.xml @@ -4,7 +4,7 @@ Promitor.Agents.Scraper - + Provides an Azure Monitor client @@ -18,7 +18,6 @@ Options for Azure Monitor integration Options for Azure Monitor logging Factory to create loggers with - Whether to use the new Azure.Monitor.Query package for queries diff --git a/src/Promitor.Agents.Scraper/Scheduling/ResourcesScrapingJob.cs b/src/Promitor.Agents.Scraper/Scheduling/ResourcesScrapingJob.cs index ce6ce8dc8..658c47466 100644 --- a/src/Promitor.Agents.Scraper/Scheduling/ResourcesScrapingJob.cs +++ b/src/Promitor.Agents.Scraper/Scheduling/ResourcesScrapingJob.cs @@ -277,10 +277,10 @@ private async Task ScrapeMetric(ScrapeDefinition scrap ? scrapeDefinition.Resource.SubscriptionId : _metricsDeclaration.AzureMetadata.SubscriptionId; var azureEnvironent = _metricsDeclaration.AzureMetadata.Cloud.GetAzureEnvironment(); - Logger.LogInformation("Parsed SDK Config {UseAzureMonitorSdk}", _metricsDeclaration.UseAzureMonitorSdk); + Logger.LogInformation("Parsed SDK Config {UseAzureMonitorSdk}", _azureMonitorIntegrationConfiguration.Value.UseAzureMonitorSdk); var azureMonitorClient = _azureMonitorClientFactory.CreateIfNotExists(_metricsDeclaration.AzureMetadata.Cloud, _metricsDeclaration.AzureMetadata.TenantId, resourceSubscriptionId, _metricSinkWriter, _azureScrapingSystemMetricsPublisher, _resourceMetricDefinitionMemoryCache, _configuration, - _azureMonitorIntegrationConfiguration, _azureMonitorLoggingConfiguration, _loggerFactory, _metricsDeclaration.UseAzureMonitorSdk); + _azureMonitorIntegrationConfiguration, _azureMonitorLoggingConfiguration, _loggerFactory); var tokenCredential = AzureAuthenticationFactory.GetTokenCredential(azureEnvironent.ManagementEndpoint, _metricsDeclaration.AzureMetadata.TenantId, AzureAuthenticationFactory.GetConfiguredAzureAuthentication(_configuration), new Uri(_metricsDeclaration.AzureMetadata.Cloud.GetAzureEnvironment().AuthenticationEndpoint)); diff --git a/src/Promitor.Integrations.AzureMonitor/Configuration/AzureMonitorIntegrationConfiguration.cs b/src/Promitor.Integrations.AzureMonitor/Configuration/AzureMonitorIntegrationConfiguration.cs index 52daf0f65..577f15979 100644 --- a/src/Promitor.Integrations.AzureMonitor/Configuration/AzureMonitorIntegrationConfiguration.cs +++ b/src/Promitor.Integrations.AzureMonitor/Configuration/AzureMonitorIntegrationConfiguration.cs @@ -3,5 +3,6 @@ public class AzureMonitorIntegrationConfiguration { public AzureMonitorHistoryConfiguration History { get; set; } = new(); + public bool UseAzureMonitorSdk { get; set; } = true; } } \ No newline at end of file diff --git a/src/Promitor.Tests.Unit/Configuration/RuntimeConfigurationUnitTest.cs b/src/Promitor.Tests.Unit/Configuration/RuntimeConfigurationUnitTest.cs index f100abcf0..0fb5a064e 100644 --- a/src/Promitor.Tests.Unit/Configuration/RuntimeConfigurationUnitTest.cs +++ b/src/Promitor.Tests.Unit/Configuration/RuntimeConfigurationUnitTest.cs @@ -96,6 +96,43 @@ public async Task RuntimeConfiguration_HasStartingFromInHoursConfiguredForAzureM Assert.Equal(expectedStartingFromInHours, runtimeConfiguration.AzureMonitor.Integration.History.StartingFromInHours); } + [Fact] + public async Task RuntimeConfiguration_HasNoNewSdkFlagForAzureMonitorIntegration_DefaultsToTrue() + { + // Arrange + var configuration = await RuntimeConfigurationGenerator.WithServerConfiguration() + .WithAzureMonitorIntegration(useAzureMonitorSdk: null) + .GenerateAsync(); + + // Act + var runtimeConfiguration = configuration.Get(); + + // Assert + Assert.NotNull(runtimeConfiguration); + Assert.NotNull(runtimeConfiguration.AzureMonitor); + Assert.NotNull(runtimeConfiguration.AzureMonitor.Integration); + Assert.True(runtimeConfiguration.AzureMonitor.Integration.UseAzureMonitorSdk); + } + + [Fact] + public async Task RuntimeConfiguration_OverrideNewSdkFlagForAzureMonitorIntegration_BecomesFalse() + { + // Arrange + var configuration = await RuntimeConfigurationGenerator.WithServerConfiguration() + .WithAzureMonitorIntegration(useAzureMonitorSdk: false) + .GenerateAsync(); + + // Act + var runtimeConfiguration = configuration.Get(); + + // Assert + Assert.NotNull(runtimeConfiguration); + Assert.NotNull(runtimeConfiguration.AzureMonitor); + Assert.NotNull(runtimeConfiguration.AzureMonitor.Integration); + Assert.False(runtimeConfiguration.AzureMonitor.Integration.UseAzureMonitorSdk); + } + + [Theory] [InlineData(true)] [InlineData(false)] diff --git a/src/Promitor.Tests.Unit/Generators/Config/RuntimeConfigurationGenerator.cs b/src/Promitor.Tests.Unit/Generators/Config/RuntimeConfigurationGenerator.cs index cbcab7220..0c4751a2b 100644 --- a/src/Promitor.Tests.Unit/Generators/Config/RuntimeConfigurationGenerator.cs +++ b/src/Promitor.Tests.Unit/Generators/Config/RuntimeConfigurationGenerator.cs @@ -235,7 +235,7 @@ public RuntimeConfigurationGenerator WithAzureMonitorLogging(bool isEnabled = tr return this; } - public RuntimeConfigurationGenerator WithAzureMonitorIntegration(int? startingFromInHours = 100) + public RuntimeConfigurationGenerator WithAzureMonitorIntegration(int? startingFromInHours = 100, bool? useAzureMonitorSdk = true) { _runtimeConfiguration.AzureMonitor ??= new AzureMonitorConfiguration(); _runtimeConfiguration.AzureMonitor.Integration ??= new AzureMonitorIntegrationConfiguration(); @@ -247,6 +247,11 @@ public RuntimeConfigurationGenerator WithAzureMonitorIntegration(int? startingFr _runtimeConfiguration.AzureMonitor.Integration.History.StartingFromInHours = startingFromInHours.Value; } + if (useAzureMonitorSdk != null) + { + _runtimeConfiguration.AzureMonitor.Integration.UseAzureMonitorSdk = useAzureMonitorSdk.Value; + } + return this; } @@ -340,6 +345,7 @@ public async Task GenerateAsync() if (_runtimeConfiguration?.AzureMonitor.Integration?.History != null) { configurationBuilder.AppendLine(" integration:"); + configurationBuilder.AppendLine($" useAzureMonitorSdk: {_runtimeConfiguration?.AzureMonitor.Integration.UseAzureMonitorSdk}"); configurationBuilder.AppendLine(" history:"); configurationBuilder.AppendLine($" startingFromInHours: {_runtimeConfiguration?.AzureMonitor.Integration.History.StartingFromInHours}"); } From 0d3fe34bf57d4f95a8e00feb248d28cc5feb7c75 Mon Sep 17 00:00:00 2001 From: xchen Date: Sun, 2 Jun 2024 18:52:51 -0700 Subject: [PATCH 4/6] remove SDK flag out of metrics declaration --- .../Configuration/Model/MetricsDeclaration.cs | 1 - .../v1/Model/MetricsDeclarationV1.cs | 1 - .../v1/Core/V1DeserializerTests.cs | 25 ------------------- 3 files changed, 27 deletions(-) diff --git a/src/Promitor.Core.Scraping/Configuration/Model/MetricsDeclaration.cs b/src/Promitor.Core.Scraping/Configuration/Model/MetricsDeclaration.cs index 5566e5af7..3c820975f 100644 --- a/src/Promitor.Core.Scraping/Configuration/Model/MetricsDeclaration.cs +++ b/src/Promitor.Core.Scraping/Configuration/Model/MetricsDeclaration.cs @@ -5,7 +5,6 @@ namespace Promitor.Core.Scraping.Configuration.Model { public class MetricsDeclaration { - public bool UseAzureMonitorSdk { get; set; } = true; public AzureMetadata AzureMetadata { get; set; } public MetricDefaults MetricDefaults { get; set; } = new MetricDefaults(); public List Metrics { get; set; } = new List(); diff --git a/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/MetricsDeclarationV1.cs b/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/MetricsDeclarationV1.cs index 0d139e719..4c4e80082 100644 --- a/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/MetricsDeclarationV1.cs +++ b/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/MetricsDeclarationV1.cs @@ -9,7 +9,6 @@ namespace Promitor.Core.Scraping.Configuration.Serialization.v1.Model public class MetricsDeclarationV1 { public string Version { get; set; } = SpecVersion.v1.ToString(); - public bool UseAzureMonitorSdk { get; set; } = true; public AzureMetadataV1 AzureMetadata { get; set; } public MetricDefaultsV1 MetricDefaults { get; set; } public IReadOnlyCollection Metrics { get; set; } diff --git a/src/Promitor.Tests.Unit/Serialization/v1/Core/V1DeserializerTests.cs b/src/Promitor.Tests.Unit/Serialization/v1/Core/V1DeserializerTests.cs index 01ed0f55f..3eee06a70 100644 --- a/src/Promitor.Tests.Unit/Serialization/v1/Core/V1DeserializerTests.cs +++ b/src/Promitor.Tests.Unit/Serialization/v1/Core/V1DeserializerTests.cs @@ -187,30 +187,5 @@ public void Deserialize_Metric_SetsMetricsNull() // Assert Assert.Null(declaration.Metrics); } - - [Fact] - public void Deserialize_NoSDKFlag_DefaultsToTrue() - { - // Arrange - var node = YamlUtils.CreateYamlNode( -@"azureMetadata: - tenantId: '123'"); - var builder = _deserializer.Deserialize(node, _errorReporter.Object); - - Assert.True(builder.UseAzureMonitorSdk); - } - - [Fact] - public void Deserialize_SdkSpecified_SetsCorrectFlag() - { - // Arrange - var yamlNode = YamlUtils.CreateYamlNode("useAzureMonitorSdk: false"); - - // Act - var builder = _deserializer.Deserialize(yamlNode, _errorReporter.Object); - - // Assert - Assert.False(builder.UseAzureMonitorSdk); - } } } From a9bfc44cd9d8ece962cb451b05b5a819aaf5676a Mon Sep 17 00:00:00 2001 From: xchen Date: Sun, 2 Jun 2024 18:58:01 -0700 Subject: [PATCH 5/6] remove SDK flag out of metrics declaration --- .../Configuration/Serialization/v1/Core/V1Deserializer.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/V1Deserializer.cs b/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/V1Deserializer.cs index 57759c1ad..5dd648d04 100644 --- a/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/V1Deserializer.cs +++ b/src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/V1Deserializer.cs @@ -20,8 +20,6 @@ public V1Deserializer(IDeserializer azureMetadataDeserializer, Map(definition => definition.Version) .IsRequired() .MapUsing(GetVersion); - Map(definition => definition.UseAzureMonitorSdk) - .WithDefault(true); Map(definition => definition.AzureMetadata) .IsRequired() .MapUsingDeserializer(azureMetadataDeserializer); From 07ce690b94f7d01819f711f7a8505cf06bcfa1ac Mon Sep 17 00:00:00 2001 From: xchen Date: Mon, 3 Jun 2024 08:34:00 -0700 Subject: [PATCH 6/6] revert github action --- .github/workflows/templates-build-push-image.yml | 11 ++++------- .../LegacyAzureMonitorClient.cs | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/templates-build-push-image.yml b/.github/workflows/templates-build-push-image.yml index 16f5d8440..60a8380f8 100644 --- a/.github/workflows/templates-build-push-image.yml +++ b/.github/workflows/templates-build-push-image.yml @@ -1,6 +1,5 @@ -name: Build and Push (Linux) on: - workflow_dispatch: + workflow_call: inputs: image_name: required: true @@ -16,8 +15,6 @@ jobs: linux: name: Build & Push (Linux) runs-on: ubuntu-latest - permissions: - packages: write steps: - name: Checkout Code uses: actions/checkout@v4 @@ -43,7 +40,7 @@ jobs: uses: docker/login-action@v3 with: registry: ghcr.io - username: hkfgo + username: tomkerkhove password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push preview image @@ -52,5 +49,5 @@ jobs: build-args: VERSION="${{ env.artifact_full_version }}" context: ./src/ file: ./src/${{ inputs.project_name }}/Dockerfile.linux - tags: ghcr.io/hkfgo/${{ env.image_commit_uri }},ghcr.io/hkfgo/${{ env.image_latest_uri }} - push: true \ No newline at end of file + tags: ${{ env.image_commit_uri }},${{ env.image_latest_uri }} + push: true diff --git a/src/Promitor.Integrations.AzureMonitor/LegacyAzureMonitorClient.cs b/src/Promitor.Integrations.AzureMonitor/LegacyAzureMonitorClient.cs index 7cedee566..0a541a60a 100644 --- a/src/Promitor.Integrations.AzureMonitor/LegacyAzureMonitorClient.cs +++ b/src/Promitor.Integrations.AzureMonitor/LegacyAzureMonitorClient.cs @@ -61,7 +61,7 @@ public LegacyAzureMonitorClient(AzureEnvironment azureCloud, string tenantId, st _resourceMetricDefinitionMemoryCache = resourceMetricDefinitionMemoryCache; _azureMonitorIntegrationConfiguration = azureMonitorIntegrationConfiguration; _logger = loggerFactory.CreateLogger(); - _logger.LogInformation("Using legacy scraper"); + _logger.LogWarning("Will use deprecated Azure Management Libraries for Metric Scraping"); _authenticatedAzureSubscription = CreateLegacyAzureClient(azureCloud, tenantId, subscriptionId, azureAuthenticationInfo, loggerFactory, metricSinkWriter, azureScrapingSystemMetricsPublisher, azureMonitorLoggingConfiguration); }