Add metrics to track log lines dropped because of logLevelDenyList

newrelic · Jul 11, 2023 · d095e35 · d095e35
1 parent 30a18a0
commit d095e35
Show file tree

Hide file tree

Showing 8 changed files with 71 additions and 2 deletions.
diff --git a/src/Agent/NewRelic/Agent/Core/Agent.cs b/src/Agent/NewRelic/Agent/Core/Agent.cs
@@ -423,6 +423,9 @@ public void RecordLogMessage(string frameworkName, object logEvent, Func<object,
             // LogLevelDenyList is already uppercase
             if (normalizedLevel != string.Empty && _configurationService.Configuration.LogLevelDenyList.Contains(normalizedLevel))
             {
+                if (_configurationService.Configuration.LogMetricsCollectorEnabled)
+                    _agentHealthReporter.IncrementLogDeniedCount(normalizedLevel);
+
                 return;
             }
 

diff --git a/src/Agent/NewRelic/Agent/Core/AgentHealth/AgentHealthReporter.cs b/src/Agent/NewRelic/Agent/Core/AgentHealth/AgentHealthReporter.cs
@@ -29,6 +29,7 @@ public class AgentHealthReporter : ConfigurationBasedService, IAgentHealthReport
         private readonly IList<RecurringLogData> _recurringLogDatas = new ConcurrentList<RecurringLogData>();
         private readonly IDictionary<AgentHealthEvent, InterlockedCounter> _agentHealthEventCounters = new Dictionary<AgentHealthEvent, InterlockedCounter>();
         private readonly ConcurrentDictionary<string, InterlockedCounter> _logLinesCountByLevel = new ConcurrentDictionary<string, InterlockedCounter>();
+        private readonly ConcurrentDictionary<string, InterlockedCounter> _logDeniedCountByLevel = new ConcurrentDictionary<string, InterlockedCounter>();
 
         private PublishMetricDelegate _publishMetricDelegate;
         private InterlockedCounter _payloadCreateSuccessCounter;
@@ -583,6 +584,22 @@ public void CollectLoggingMetrics()
                     _loggingForwardingEnabledWithFrameworksReported[kvp.Key] = true;
                 }
             }
+
+            var totalDeniedCount = 0;
+            foreach (var logLinesDeniedCounter in _logDeniedCountByLevel)
+            {
+                if (TryGetCount(logLinesDeniedCounter.Value, out var linesCount))
+                {
+                    totalDeniedCount += linesCount;
+                    TrySend(_metricBuilder.TryBuildLoggingMetricsDeniedCountBySeverityMetric(logLinesDeniedCounter.Key, linesCount));
+                }
+            }
+
+            if (totalDeniedCount > 0)
+            {
+                TrySend(_metricBuilder.TryBuildLoggingMetricsDeniedCountMetric(totalDeniedCount));
+            }
+
         }
 
         public void IncrementLogLinesCount(string level)
@@ -591,6 +608,12 @@ public void IncrementLogLinesCount(string level)
             _logLinesCountByLevel[level].Increment();
         }
 
+        public void IncrementLogDeniedCount(string level)
+        {
+            _logDeniedCountByLevel.TryAdd(level, new InterlockedCounter());
+            _logDeniedCountByLevel[level].Increment();
+        }
+
         public void ReportLoggingEventCollected() => TrySend(_metricBuilder.TryBuildSupportabilityLoggingEventsCollectedMetric());
 
         public void ReportLoggingEventsSent(int count) => TrySend(_metricBuilder.TryBuildSupportabilityLoggingEventsSentMetric(count));

diff --git a/src/Agent/NewRelic/Agent/Core/AgentHealth/IAgentHealthReporter.cs b/src/Agent/NewRelic/Agent/Core/AgentHealth/IAgentHealthReporter.cs
@@ -142,6 +142,7 @@ public interface IAgentHealthReporter : IOutOfBandMetricSource
         void ReportSupportabilityDataUsage(string api, string apiArea, long dataSent, long dataReceived);
 
         void IncrementLogLinesCount(string logLevel);
+        void IncrementLogDeniedCount(string logLevel);
         void ReportLoggingEventCollected();
         void ReportLoggingEventsSent(int count);
         void ReportLoggingEventsDropped(int droppedCount);

diff --git a/src/Agent/NewRelic/Agent/Core/NewRelic.Agent.Core.Metric/MetricNames.cs b/src/Agent/NewRelic/Agent/Core/NewRelic.Agent.Core.Metric/MetricNames.cs
@@ -1040,6 +1040,7 @@ public static string GetPerDestinationAreaDataUsageMetricName(string destination
 
         private const string LoggingMetrics = "Logging";
         private const string LoggingMetricsDotnetLines = LoggingMetrics + PathSeparator + "lines";
+        private const string LoggingMetricsDotnetDenied = LoggingMetrics + PathSeparator + "denied";
         private const string SupportabilityLoggingEventsPs = SupportabilityPs + "Logging" + PathSeparator;
         public const string SupportabilityLoggingEventsSent = SupportabilityLoggingEventsPs + Forwarding + PathSeparator + "Sent";
         public const string SupportabilityLoggingEventsCollected = SupportabilityLoggingEventsPs + Forwarding + PathSeparator + "Seen";
@@ -1055,6 +1056,16 @@ public static string GetLoggingMetricsLinesName()
             return LoggingMetricsDotnetLines;
         }
 
+        public static string GetLoggingMetricsDeniedBySeverityName(string logLevel)
+        {
+            return LoggingMetricsDotnetDenied + PathSeparator + logLevel;
+        }
+
+        public static string GetLoggingMetricsDeniedName()
+        {
+            return LoggingMetricsDotnetDenied;
+        }
+
         private const string Enabled = "enabled";
         private const string Disabled = "disabled";
         private const string Metrics = "Metrics";

diff --git a/src/Agent/NewRelic/Agent/Core/WireModels/IMetricBuilder.cs b/src/Agent/NewRelic/Agent/Core/WireModels/IMetricBuilder.cs
@@ -196,6 +196,10 @@ public interface IMetricBuilder
 
         MetricWireModel TryBuildLoggingMetricsLinesCountMetric(int count);
 
+        MetricWireModel TryBuildLoggingMetricsDeniedCountBySeverityMetric(string logLevel, int count);
+
+        MetricWireModel TryBuildLoggingMetricsDeniedCountMetric(int count);
+
         MetricWireModel TryBuildSupportabilityLoggingEventsCollectedMetric();
 
         MetricWireModel TryBuildSupportabilityLoggingEventsSentMetric(int loggingEventCount);

diff --git a/src/Agent/NewRelic/Agent/Core/WireModels/MetricWireModel.cs b/src/Agent/NewRelic/Agent/Core/WireModels/MetricWireModel.cs
@@ -955,6 +955,18 @@ public MetricWireModel TryBuildLoggingMetricsLinesCountMetric(int count)
                 return BuildMetric(_metricNameService, proposedName, null, MetricDataWireModel.BuildCountData(count));
             }
 
+            public MetricWireModel TryBuildLoggingMetricsDeniedCountBySeverityMetric(string logLevel, int count)
+            {
+                var proposedName = MetricNames.GetLoggingMetricsDeniedBySeverityName(logLevel);
+                return BuildMetric(_metricNameService, proposedName, null, MetricDataWireModel.BuildCountData(count));
+            }
+
+            public MetricWireModel TryBuildLoggingMetricsDeniedCountMetric(int count)
+            {
+                var proposedName = MetricNames.GetLoggingMetricsDeniedName();
+                return BuildMetric(_metricNameService, proposedName, null, MetricDataWireModel.BuildCountData(count));
+            }
+
             public MetricWireModel TryBuildSupportabilityLoggingEventsCollectedMetric()
             {
                 const string proposedName = MetricNames.SupportabilityLoggingEventsCollected;

diff --git a/tests/Agent/IntegrationTests/IntegrationTests/Logging/LogLevelDenyListTests.cs b/tests/Agent/IntegrationTests/IntegrationTests/Logging/LogLevelDenyListTests.cs
@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using MultiFunctionApplicationHelpers;
 using NewRelic.Agent.IntegrationTestHelpers;
 using Xunit;
@@ -56,22 +57,27 @@ public logLevelDenyListTestsBase(TFixture fixture, ITestOutputHelper output,
         }
 
         [Fact]
-        public void LogLinesPerLevelMetricsExist()
+        public void LoggingMetricsExist()
         {
             var expectedMetrics = new List<Assertions.ExpectedMetric>
             {
                 new Assertions.ExpectedMetric { metricName = "Logging/lines/" + LogUtils.GetLevelName(_loggingFramework, "WARN"), callCount = 1 },
                 new Assertions.ExpectedMetric { metricName = "Logging/lines/" + LogUtils.GetLevelName(_loggingFramework, "ERROR"), callCount = 1 },
 
                 new Assertions.ExpectedMetric { metricName = "Logging/lines", callCount = 2 },
+
+                new Assertions.ExpectedMetric { metricName = "Logging/denied/" + LogUtils.GetLevelName(_loggingFramework, "DEBUG"), callCount = 1 },
+                new Assertions.ExpectedMetric { metricName = "Logging/denied/" + LogUtils.GetLevelName(_loggingFramework, "INFO"), callCount = 1 },
+
+                new Assertions.ExpectedMetric { metricName = "Logging/denied", callCount = 2 },
             };
             var notExpectedMetrics = new List<Assertions.ExpectedMetric>
             {
                 new Assertions.ExpectedMetric { metricName = "Logging/lines/" + LogUtils.GetLevelName(_loggingFramework, "DEBUG") },
                 new Assertions.ExpectedMetric { metricName = "Logging/lines/" + LogUtils.GetLevelName(_loggingFramework, "INFO") }
             };
 
-            var actualMetrics = _fixture.AgentLog.GetMetrics();
+            var actualMetrics = _fixture.AgentLog.GetMetrics().ToList();
             Assertions.MetricsExist(expectedMetrics, actualMetrics);
             Assertions.MetricsDoNotExist(notExpectedMetrics, actualMetrics);
         }

diff --git a/tests/Agent/UnitTests/Core.UnitTest/Metrics/MetricNamesTests.cs b/tests/Agent/UnitTests/Core.UnitTest/Metrics/MetricNamesTests.cs
@@ -207,6 +207,15 @@ public static void MetricNamesTest_Events()
                 Is.EqualTo("Supportability/Events/Customer/TryResizeReservoir"));
         }
 
+        [Test]
+        public static void MetricNamesTest_Logging()
+        {
+            Assert.That(MetricNames.GetLoggingMetricsLinesName(), Is.EqualTo("Logging/lines"));
+            Assert.That(MetricNames.GetLoggingMetricsDeniedName(), Is.EqualTo("Logging/denied"));
+            Assert.That(MetricNames.GetLoggingMetricsLinesBySeverityName("foo"), Is.EqualTo("Logging/lines/foo"));
+            Assert.That(MetricNames.GetLoggingMetricsDeniedBySeverityName("foo"), Is.EqualTo("Logging/denied/foo"));
+        }
+
         [Test]
         public static void MetricNamesTest_AnalyticEvents()
         {