From 1a833555547166a6a73e532ebc48b5344837ab2a Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 9 Mar 2021 15:15:17 -0600 Subject: [PATCH 1/8] added StatsD App.Metrics support --- .../Petabridge.Phobos.Web.DataDog.csproj | 8 ++++---- src/Petabridge.Phobos.Web.DataDog/Startup.cs | 17 +++++++++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/Petabridge.Phobos.Web.DataDog/Petabridge.Phobos.Web.DataDog.csproj b/src/Petabridge.Phobos.Web.DataDog/Petabridge.Phobos.Web.DataDog.csproj index 828be7e..b5423a3 100644 --- a/src/Petabridge.Phobos.Web.DataDog/Petabridge.Phobos.Web.DataDog.csproj +++ b/src/Petabridge.Phobos.Web.DataDog/Petabridge.Phobos.Web.DataDog.csproj @@ -15,7 +15,9 @@ - + + + @@ -25,9 +27,7 @@ 0.5.0 - - - + \ No newline at end of file diff --git a/src/Petabridge.Phobos.Web.DataDog/Startup.cs b/src/Petabridge.Phobos.Web.DataDog/Startup.cs index bb90683..b7d9b12 100644 --- a/src/Petabridge.Phobos.Web.DataDog/Startup.cs +++ b/src/Petabridge.Phobos.Web.DataDog/Startup.cs @@ -12,6 +12,7 @@ using Akka.Bootstrap.Docker; using Akka.Configuration; using App.Metrics; +using App.Metrics.Reporting.StatsD.Builder; using Datadog.Trace.OpenTracing; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; @@ -81,7 +82,16 @@ public static void ConfigureAppMetrics(IServiceCollection services) o.Enabled = true; o.ReportingEnabled = true; }) - .Report.ToDatadogHttp(options => { + .Report.ToStatsDUdp(b => { + b.SocketSettings.Address = {Environment.GetEnvironmentVariable("DD_AGENT_HOST"); + b.SocketSettings.Port = 8125; + }) + .Build(); + }); + services.AddMetricsReportingHostedService(); + +/* + .ToDatadogHttp(options => { // need to wait until we have DogstatsD support https://github.com/AppMetrics/AppMetrics/pull/627 //options.Datadog.BaseUri = new Uri($"http://{Environment.GetEnvironmentVariable("DD_AGENT_HOST")}"); options.Datadog.BaseUri = new Uri($"https://api.datadoghq.com/"); @@ -91,9 +101,8 @@ public static void ConfigureAppMetrics(IServiceCollection services) options.HttpPolicy.Timeout = TimeSpan.FromSeconds(10); options.FlushInterval = TimeSpan.FromSeconds(20); }) - .Build(); - }); - services.AddMetricsReportingHostedService(); +*/ + } public static void ConfigureDataDogTracing(IServiceCollection services) From a9452b38d198b0f01bf7d3040a3752f90625eedb Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 9 Mar 2021 15:18:38 -0600 Subject: [PATCH 2/8] fixed some small issues --- README.md | 2 +- src/Petabridge.Phobos.Web.DataDog/Startup.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b12a438..8d0fc9b 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ This sample is designed to work with [DataDog Application Performance Monitoring To run this sample, you will need a copy of your DataDog API Key - which will be passed into the local Kubernetes cluster created by this tutorial. -To [get your DataDog API Key for use with DataDog's Helm Agent, click here](https://app.datadoghq.com/account/settings#agent/kubernetes). +**To [get your DataDog API Key for use with DataDog's Helm Agent, click here](https://app.datadoghq.com/account/settings#agent/kubernetes)**. ### Retreiving the DataDog Helm Charts This sample uses [Helm](https://helm.sh/) to install [DataDog's Agent Helm Chart](https://github.com/DataDog/helm-charts/blob/master/charts/datadog/README.md) into a local Kubernetes cluster. diff --git a/src/Petabridge.Phobos.Web.DataDog/Startup.cs b/src/Petabridge.Phobos.Web.DataDog/Startup.cs index b7d9b12..be23529 100644 --- a/src/Petabridge.Phobos.Web.DataDog/Startup.cs +++ b/src/Petabridge.Phobos.Web.DataDog/Startup.cs @@ -83,7 +83,7 @@ public static void ConfigureAppMetrics(IServiceCollection services) o.ReportingEnabled = true; }) .Report.ToStatsDUdp(b => { - b.SocketSettings.Address = {Environment.GetEnvironmentVariable("DD_AGENT_HOST"); + b.SocketSettings.Address = Environment.GetEnvironmentVariable("DD_AGENT_HOST"); b.SocketSettings.Port = 8125; }) .Build(); From 76265ae0593d9962d8538db78bc3ddf12c3b5b9c Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 9 Mar 2021 15:25:06 -0600 Subject: [PATCH 3/8] remembered to use the DogStatsD formatter --- src/Petabridge.Phobos.Web.DataDog/Startup.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Petabridge.Phobos.Web.DataDog/Startup.cs b/src/Petabridge.Phobos.Web.DataDog/Startup.cs index be23529..d29fbc0 100644 --- a/src/Petabridge.Phobos.Web.DataDog/Startup.cs +++ b/src/Petabridge.Phobos.Web.DataDog/Startup.cs @@ -12,6 +12,7 @@ using Akka.Bootstrap.Docker; using Akka.Configuration; using App.Metrics; +using App.Metrics.Formatting.StatsD; using App.Metrics.Reporting.StatsD.Builder; using Datadog.Trace.OpenTracing; using Microsoft.AspNetCore.Builder; @@ -85,6 +86,7 @@ public static void ConfigureAppMetrics(IServiceCollection services) .Report.ToStatsDUdp(b => { b.SocketSettings.Address = Environment.GetEnvironmentVariable("DD_AGENT_HOST"); b.SocketSettings.Port = 8125; + b.StatsDOptions.MetricNameFormatter = new DefaultDogStatsDMetricStringSerializer(); }) .Build(); }); From 24ccbb4185a5c22a93fbc7ccd6bca863f26caaf3 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 9 Mar 2021 15:15:17 -0600 Subject: [PATCH 4/8] added StatsD App.Metrics support --- .../Petabridge.Phobos.Web.DataDog.csproj | 8 ++++---- src/Petabridge.Phobos.Web.DataDog/Startup.cs | 17 +++++++++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/Petabridge.Phobos.Web.DataDog/Petabridge.Phobos.Web.DataDog.csproj b/src/Petabridge.Phobos.Web.DataDog/Petabridge.Phobos.Web.DataDog.csproj index 06e4153..ea5084f 100644 --- a/src/Petabridge.Phobos.Web.DataDog/Petabridge.Phobos.Web.DataDog.csproj +++ b/src/Petabridge.Phobos.Web.DataDog/Petabridge.Phobos.Web.DataDog.csproj @@ -15,8 +15,10 @@ - + + + @@ -25,9 +27,7 @@ 0.5.0 - - - + \ No newline at end of file diff --git a/src/Petabridge.Phobos.Web.DataDog/Startup.cs b/src/Petabridge.Phobos.Web.DataDog/Startup.cs index bb90683..b7d9b12 100644 --- a/src/Petabridge.Phobos.Web.DataDog/Startup.cs +++ b/src/Petabridge.Phobos.Web.DataDog/Startup.cs @@ -12,6 +12,7 @@ using Akka.Bootstrap.Docker; using Akka.Configuration; using App.Metrics; +using App.Metrics.Reporting.StatsD.Builder; using Datadog.Trace.OpenTracing; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; @@ -81,7 +82,16 @@ public static void ConfigureAppMetrics(IServiceCollection services) o.Enabled = true; o.ReportingEnabled = true; }) - .Report.ToDatadogHttp(options => { + .Report.ToStatsDUdp(b => { + b.SocketSettings.Address = {Environment.GetEnvironmentVariable("DD_AGENT_HOST"); + b.SocketSettings.Port = 8125; + }) + .Build(); + }); + services.AddMetricsReportingHostedService(); + +/* + .ToDatadogHttp(options => { // need to wait until we have DogstatsD support https://github.com/AppMetrics/AppMetrics/pull/627 //options.Datadog.BaseUri = new Uri($"http://{Environment.GetEnvironmentVariable("DD_AGENT_HOST")}"); options.Datadog.BaseUri = new Uri($"https://api.datadoghq.com/"); @@ -91,9 +101,8 @@ public static void ConfigureAppMetrics(IServiceCollection services) options.HttpPolicy.Timeout = TimeSpan.FromSeconds(10); options.FlushInterval = TimeSpan.FromSeconds(20); }) - .Build(); - }); - services.AddMetricsReportingHostedService(); +*/ + } public static void ConfigureDataDogTracing(IServiceCollection services) From e40e31202fb6ec1dae1c8a8c7d49362c38a7fa1d Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 9 Mar 2021 15:18:38 -0600 Subject: [PATCH 5/8] fixed some small issues --- README.md | 2 +- src/Petabridge.Phobos.Web.DataDog/Startup.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b12a438..8d0fc9b 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ This sample is designed to work with [DataDog Application Performance Monitoring To run this sample, you will need a copy of your DataDog API Key - which will be passed into the local Kubernetes cluster created by this tutorial. -To [get your DataDog API Key for use with DataDog's Helm Agent, click here](https://app.datadoghq.com/account/settings#agent/kubernetes). +**To [get your DataDog API Key for use with DataDog's Helm Agent, click here](https://app.datadoghq.com/account/settings#agent/kubernetes)**. ### Retreiving the DataDog Helm Charts This sample uses [Helm](https://helm.sh/) to install [DataDog's Agent Helm Chart](https://github.com/DataDog/helm-charts/blob/master/charts/datadog/README.md) into a local Kubernetes cluster. diff --git a/src/Petabridge.Phobos.Web.DataDog/Startup.cs b/src/Petabridge.Phobos.Web.DataDog/Startup.cs index b7d9b12..be23529 100644 --- a/src/Petabridge.Phobos.Web.DataDog/Startup.cs +++ b/src/Petabridge.Phobos.Web.DataDog/Startup.cs @@ -83,7 +83,7 @@ public static void ConfigureAppMetrics(IServiceCollection services) o.ReportingEnabled = true; }) .Report.ToStatsDUdp(b => { - b.SocketSettings.Address = {Environment.GetEnvironmentVariable("DD_AGENT_HOST"); + b.SocketSettings.Address = Environment.GetEnvironmentVariable("DD_AGENT_HOST"); b.SocketSettings.Port = 8125; }) .Build(); From bd0c0fa318a2d28cb0ae68cff77f9126c6546b90 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 9 Mar 2021 15:25:06 -0600 Subject: [PATCH 6/8] remembered to use the DogStatsD formatter --- src/Petabridge.Phobos.Web.DataDog/Startup.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Petabridge.Phobos.Web.DataDog/Startup.cs b/src/Petabridge.Phobos.Web.DataDog/Startup.cs index be23529..d29fbc0 100644 --- a/src/Petabridge.Phobos.Web.DataDog/Startup.cs +++ b/src/Petabridge.Phobos.Web.DataDog/Startup.cs @@ -12,6 +12,7 @@ using Akka.Bootstrap.Docker; using Akka.Configuration; using App.Metrics; +using App.Metrics.Formatting.StatsD; using App.Metrics.Reporting.StatsD.Builder; using Datadog.Trace.OpenTracing; using Microsoft.AspNetCore.Builder; @@ -85,6 +86,7 @@ public static void ConfigureAppMetrics(IServiceCollection services) .Report.ToStatsDUdp(b => { b.SocketSettings.Address = Environment.GetEnvironmentVariable("DD_AGENT_HOST"); b.SocketSettings.Port = 8125; + b.StatsDOptions.MetricNameFormatter = new DefaultDogStatsDMetricStringSerializer(); }) .Build(); }); From 218f27d23beaf6205d06bca18e95eb8219c4d7f7 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Wed, 31 Mar 2021 11:27:43 -0500 Subject: [PATCH 7/8] upgraded to App.Metrics v4.2.0-preview0.34 --- .../Petabridge.Phobos.Web.DataDog.csproj | 6 +++--- src/Petabridge.Phobos.Web.DataDog/Startup.cs | 2 +- src/common.props | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Petabridge.Phobos.Web.DataDog/Petabridge.Phobos.Web.DataDog.csproj b/src/Petabridge.Phobos.Web.DataDog/Petabridge.Phobos.Web.DataDog.csproj index ea5084f..b3721a6 100644 --- a/src/Petabridge.Phobos.Web.DataDog/Petabridge.Phobos.Web.DataDog.csproj +++ b/src/Petabridge.Phobos.Web.DataDog/Petabridge.Phobos.Web.DataDog.csproj @@ -15,10 +15,10 @@ - + - - + + diff --git a/src/Petabridge.Phobos.Web.DataDog/Startup.cs b/src/Petabridge.Phobos.Web.DataDog/Startup.cs index d29fbc0..0c34a59 100644 --- a/src/Petabridge.Phobos.Web.DataDog/Startup.cs +++ b/src/Petabridge.Phobos.Web.DataDog/Startup.cs @@ -13,7 +13,7 @@ using Akka.Configuration; using App.Metrics; using App.Metrics.Formatting.StatsD; -using App.Metrics.Reporting.StatsD.Builder; +using App.Metrics.Reporting.StatsD; using Datadog.Trace.OpenTracing; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; diff --git a/src/common.props b/src/common.props index 6054957..82d18a3 100644 --- a/src/common.props +++ b/src/common.props @@ -2,7 +2,7 @@ Copyright © 2019-2021 Petabridge Petabridge - 0.1.0 + 0.1.1 First release @@ -23,4 +23,4 @@ 5.10.3 1.1.1 - + \ No newline at end of file From 9920391cccbb10c167a9ccf8904c0a62b6213a6b Mon Sep 17 00:00:00 2001 From: Gregorius Soedharmo Date: Sat, 8 May 2021 05:55:17 +0700 Subject: [PATCH 8/8] Adjust code to support DataDog logging --- k8s/datadog/datadog-values.yaml | 4 +-- k8s/environment/configs/configs.yaml | 6 +++- .../petabridge.phobos.web.datadog.yaml | 28 +++++++++++++++++++ src/Petabridge.Phobos.Web.DataDog/Dockerfile | 17 +++++++++++ .../Petabridge.Phobos.Web.DataDog.csproj | 1 - src/Petabridge.Phobos.Web.DataDog/Startup.cs | 25 +++-------------- src/Petabridge.Phobos.Web.DataDog/app.conf | 1 + 7 files changed, 57 insertions(+), 25 deletions(-) diff --git a/k8s/datadog/datadog-values.yaml b/k8s/datadog/datadog-values.yaml index 5808d86..e8f2f55 100644 --- a/k8s/datadog/datadog-values.yaml +++ b/k8s/datadog/datadog-values.yaml @@ -156,11 +156,11 @@ datadog: logs: # datadog.logs.enabled -- Enables this to activate Datadog Agent log collection ## ref: https://docs.datadoghq.com/agent/basic_agent_usage/kubernetes/#log-collection-setup - enabled: false + enabled: true # datadog.logs.containerCollectAll -- Enable this to allow log collection for all containers ## ref: https://docs.datadoghq.com/agent/basic_agent_usage/kubernetes/#log-collection-setup - containerCollectAll: false + containerCollectAll: true # datadog.logs.containerCollectUsingFiles -- Collect logs from files in /var/log/pods instead of using container runtime API ## It's usually the most efficient way of collecting logs. diff --git a/k8s/environment/configs/configs.yaml b/k8s/environment/configs/configs.yaml index f19c6b1..82114f7 100644 --- a/k8s/environment/configs/configs.yaml +++ b/k8s/environment/configs/configs.yaml @@ -5,4 +5,8 @@ metadata: name: pb-configs data: # Configuration values can be set as key-value properties - environment: Development \ No newline at end of file + environment: "Development" + dd-service: "akka.cluster" + dd-dogstatsd-port: "8125" + dd-trace-agent-port: "8126" + dd-apm-receiver-port: "8126" \ No newline at end of file diff --git a/k8s/services/petabridge.phobos.web.datadog.yaml b/k8s/services/petabridge.phobos.web.datadog.yaml index ca3a56d..6e2d04c 100644 --- a/k8s/services/petabridge.phobos.web.datadog.yaml +++ b/k8s/services/petabridge.phobos.web.datadog.yaml @@ -72,10 +72,38 @@ spec: configMapKeyRef: name: pb-configs key: environment + - name: DD_ENV + valueFrom: + fieldRef: + fieldPath: metadata.labels['tags.datadoghq.com/env'] + - name: DD_SERVICE + valueFrom: + configMapKeyRef: + name: pb-configs + key: dd-service + - name: DD_VERSION + valueFrom: + fieldRef: + fieldPath: metadata.labels['tags.datadoghq.com/version'] - name: DD_AGENT_HOST valueFrom: fieldRef: fieldPath: status.hostIP + - name: DD_DOGSTATSD_PORT + valueFrom: + configMapKeyRef: + name: pb-configs + key: dd-dogstatsd-port + - name: DD_TRACE_AGENT_PORT + valueFrom: + configMapKeyRef: + name: pb-configs + key: dd-trace-agent-port + - name: DD_APM_RECEIVER_PORT + valueFrom: + configMapKeyRef: + name: pb-configs + key: dd-apm-receiver-port - name: DD_API_KEY valueFrom: secretKeyRef: diff --git a/src/Petabridge.Phobos.Web.DataDog/Dockerfile b/src/Petabridge.Phobos.Web.DataDog/Dockerfile index 09837dd..aeb12d0 100644 --- a/src/Petabridge.Phobos.Web.DataDog/Dockerfile +++ b/src/Petabridge.Phobos.Web.DataDog/Dockerfile @@ -30,4 +30,21 @@ COPY --from=base /root/.dotnet /root/.dotnet/ # Needed because https://stackoverflow.com/questions/51977474/install-dotnet-core-tool-dockerfile ENV PATH="${PATH}:/root/.dotnet/tools" +# Please select the corresponding download for your Linux containers https://github.com/DataDog/dd-trace-dotnet/releases/latest + +# Download and install the Tracer +RUN mkdir -p /opt/datadog \ + && mkdir -p /var/log/datadog \ + && TRACER_VERSION=$(curl -s https://api.github.com/repos/DataDog/dd-trace-dotnet/releases/latest | grep tag_name | cut -d '"' -f 4 | cut -c2-) \ + && curl -LO https://github.com/DataDog/dd-trace-dotnet/releases/download/v${TRACER_VERSION}/datadog-dotnet-apm_${TRACER_VERSION}_amd64.deb \ + && dpkg -i ./datadog-dotnet-apm_${TRACER_VERSION}_amd64.deb \ + && rm ./datadog-dotnet-apm_${TRACER_VERSION}_amd64.deb + +# Enable the tracer +ENV CORECLR_ENABLE_PROFILING=1 +ENV CORECLR_PROFILER={846F5F1C-F9AE-4B07-969E-05C26BC060D8} +ENV CORECLR_PROFILER_PATH=/opt/datadog/Datadog.Trace.ClrProfiler.Native.so +ENV DD_DOTNET_TRACER_HOME=/opt/datadog +ENV DD_INTEGRATIONS=/opt/datadog/integrations.json + CMD ["dotnet", "Petabridge.Phobos.Web.DataDog.dll"] \ No newline at end of file diff --git a/src/Petabridge.Phobos.Web.DataDog/Petabridge.Phobos.Web.DataDog.csproj b/src/Petabridge.Phobos.Web.DataDog/Petabridge.Phobos.Web.DataDog.csproj index b29e318..43e6da6 100644 --- a/src/Petabridge.Phobos.Web.DataDog/Petabridge.Phobos.Web.DataDog.csproj +++ b/src/Petabridge.Phobos.Web.DataDog/Petabridge.Phobos.Web.DataDog.csproj @@ -16,7 +16,6 @@ - diff --git a/src/Petabridge.Phobos.Web.DataDog/Startup.cs b/src/Petabridge.Phobos.Web.DataDog/Startup.cs index 0c34a59..a998197 100644 --- a/src/Petabridge.Phobos.Web.DataDog/Startup.cs +++ b/src/Petabridge.Phobos.Web.DataDog/Startup.cs @@ -85,35 +85,19 @@ public static void ConfigureAppMetrics(IServiceCollection services) }) .Report.ToStatsDUdp(b => { b.SocketSettings.Address = Environment.GetEnvironmentVariable("DD_AGENT_HOST"); - b.SocketSettings.Port = 8125; + b.SocketSettings.Port = int.Parse(Environment.GetEnvironmentVariable("DD_DOGSTATSD_PORT")); b.StatsDOptions.MetricNameFormatter = new DefaultDogStatsDMetricStringSerializer(); }) .Build(); }); services.AddMetricsReportingHostedService(); - -/* - .ToDatadogHttp(options => { - // need to wait until we have DogstatsD support https://github.com/AppMetrics/AppMetrics/pull/627 - //options.Datadog.BaseUri = new Uri($"http://{Environment.GetEnvironmentVariable("DD_AGENT_HOST")}"); - options.Datadog.BaseUri = new Uri($"https://api.datadoghq.com/"); - options.Datadog.ApiKey = Environment.GetEnvironmentVariable("DD_API_KEY"); - options.HttpPolicy.BackoffPeriod = TimeSpan.FromSeconds(30); - options.HttpPolicy.FailuresBeforeBackoff = 5; - options.HttpPolicy.Timeout = TimeSpan.FromSeconds(10); - options.FlushInterval = TimeSpan.FromSeconds(20); - }) -*/ - } public static void ConfigureDataDogTracing(IServiceCollection services) { // Add DataDog Tracing - services.AddSingleton(sp => - { - return OpenTracingTracerFactory.CreateTracer().WithScopeManager(new ActorScopeManager()); - }); + var tracer = OpenTracingTracerFactory.CreateTracer().WithScopeManager(new ActorScopeManager()); + services.AddSingleton(tracer); } public static void ConfigureAkka(IServiceCollection services) @@ -131,8 +115,7 @@ public static void ConfigureAkka(IServiceCollection services) .WithTracing(t => t.SetTracer(tracer))) // binds Phobos to same tracer as ASP.NET Core .WithSetup(BootstrapSetup.Create() .WithConfig(config) // passes in the HOCON for Akka.NET to the ActorSystem - .WithActorRefProvider(PhobosProviderSelection - .Cluster)); // last line activates Phobos inside Akka.NET + .WithActorRefProvider(PhobosProviderSelection.Cluster)); // last line activates Phobos inside Akka.NET var sys = ActorSystem.Create("ClusterSys", phobosSetup); diff --git a/src/Petabridge.Phobos.Web.DataDog/app.conf b/src/Petabridge.Phobos.Web.DataDog/app.conf index 0a98a37..2404a22 100644 --- a/src/Petabridge.Phobos.Web.DataDog/app.conf +++ b/src/Petabridge.Phobos.Web.DataDog/app.conf @@ -15,6 +15,7 @@ petabridge.cmd{ } akka { + loggers = ["Akka.Event.DefaultLogger", "Phobos.Tracing.DataDogLogger, Phobos.Tracing"] actor { provider = cluster