From 515afc254086944137377ed8dcf48233e3f32692 Mon Sep 17 00:00:00 2001 From: Maksym Ochenashko Date: Fri, 13 Sep 2024 21:34:14 +0300 Subject: [PATCH] sdk-contrib: add `AWSEC2Detector` --- build.sbt | 11 +- docs/sdk/aws-resource-detectors.md | 75 +++++- .../contrib/aws/resource/AWSEC2Detector.scala | 220 ++++++++++++++++++ .../aws/resource/AWSEC2DetectorSuite.scala | 115 +++++++++ 4 files changed, 415 insertions(+), 6 deletions(-) create mode 100644 sdk-contrib/aws/resource/src/main/scala/org/typelevel/otel4s/sdk/contrib/aws/resource/AWSEC2Detector.scala create mode 100644 sdk-contrib/aws/resource/src/test/scala/org/typelevel/otel4s/sdk/contrib/aws/resource/AWSEC2DetectorSuite.scala diff --git a/build.sbt b/build.sbt index 3723972e3..7a5982598 100644 --- a/build.sbt +++ b/build.sbt @@ -446,10 +446,18 @@ lazy val `sdk-contrib-aws-resource` = .dependsOn(`sdk-common`, `semconv-experimental` % Test) .settings( name := "otel4s-sdk-contrib-aws-resource", - startYear := Some(2024) + startYear := Some(2024), + libraryDependencies ++= Seq( + "org.http4s" %%% "http4s-ember-client" % Http4sVersion, + "org.http4s" %%% "http4s-circe" % Http4sVersion, + "org.http4s" %%% "http4s-dsl" % Http4sVersion % Test + ) ) .settings(munitDependencies) .settings(scalafixSettings) + .jsSettings(scalaJSLinkerSettings) + .nativeEnablePlugins(ScalaNativeBrewedConfigPlugin) + .nativeSettings(scalaNativeSettings) lazy val `sdk-contrib-aws-xray-propagator` = crossProject(JVMPlatform, JSPlatform, NativePlatform) @@ -735,6 +743,7 @@ lazy val docs = project libraryDependencies ++= Seq( "org.apache.pekko" %% "pekko-http" % PekkoHttpVersion, "org.http4s" %% "http4s-client" % Http4sVersion, + "org.http4s" %% "http4s-dsl" % Http4sVersion, "io.opentelemetry" % "opentelemetry-sdk-extension-autoconfigure" % OpenTelemetryVersion, "io.opentelemetry.instrumentation" % "opentelemetry-instrumentation-annotations" % OpenTelemetryInstrumentationVersion, "io.opentelemetry.instrumentation" % "opentelemetry-runtime-telemetry-java8" % OpenTelemetryInstrumentationAlphaVersion, diff --git a/docs/sdk/aws-resource-detectors.md b/docs/sdk/aws-resource-detectors.md index fbdfa6398..3405838d4 100644 --- a/docs/sdk/aws-resource-detectors.md +++ b/docs/sdk/aws-resource-detectors.md @@ -1,6 +1,6 @@ # AWS | Resource detectors -Resource detectors can add environment-specific attributes to the telemetry resource. +Resource detectors can add environment-specific attributes to the telemetry resource. AWS detectors are implemented as a third-party library, and you need to enable them manually. ## The list of detectors @@ -45,6 +45,67 @@ AWSLambdaDetector[IO].detect.unsafeRunSync().foreach { resource => println("```") ``` +### 3. aws-ec2 + +The detector fetches instance metadata from the `http://169.254.169.254` endpoint. +See [AWS documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html) for more +details. + +```scala mdoc:reset:passthrough +import cats.effect.IO +import cats.effect.unsafe.implicits.global +import io.circe.Json +import io.circe.syntax._ +import org.http4s._ +import org.http4s.circe.jsonEncoder +import org.http4s.client.Client +import org.http4s.dsl.io._ +import org.http4s.syntax.literals._ +import org.typelevel.otel4s.sdk.contrib.aws.resource._ + +val hostname = "ip-10-0-0-1.eu-west-1.compute.internal" +val metadata = Json.obj( + "accountId" := "1234567890", + "architecture" := "x86_64", + "availabilityZone" := "eu-west-1a", + "imageId" := "ami-abc123de", + "instanceId" := "i-abc321de", + "instanceType" := "t3.small", + "privateIp" := "10.0.0.1", + "region" := "eu-west-1", + "version" := "2017-09-30" +) + +val client = Client.fromHttpApp[IO]( + HttpRoutes + .of[IO] { + case GET -> Root / "latest" / "meta-data" / "hostname" => Ok(hostname) + case GET -> Root / "latest" / "dynamic" / "instance-identity" / "document" => Ok(metadata) + case PUT -> Root / "latest" / "api" / "token" => Ok("token") + } + .orNotFound +) + +println("The `http://169.254.169.254/latest/dynamic/instance-identity/document` response: ") +println("```json") +println(metadata) +println("```") + +println("The `http://169.254.169.254/latest/meta-data/hostname` response:") +println("```") +println(hostname) +println("```") + +println("Detected resource: ") +println("```") +AWSEC2Detector[IO](uri"", client).detect.unsafeRunSync().foreach { resource => + resource.attributes.toList.sortBy(_.key.name).foreach { attribute => + println(attribute.key.name + ": " + attribute.value) + } +} +println("```") +``` + ## Getting Started @:select(build-tool) @@ -104,6 +165,8 @@ object TelemetryApp extends IOApp.Simple { _.addExportersConfigurer(OtlpExportersAutoConfigure[IO]) // register AWS Lambda detector .addResourceDetector(AWSLambdaDetector[IO]) + // register AWS EC2 detector + .addResourceDetector(AWSEC2Detector[IO]) ) .use { autoConfigured => val sdk = autoConfigured.sdk @@ -138,6 +201,8 @@ object TelemetryApp extends IOApp.Simple { _.addExporterConfigurer(OtlpSpanExporterAutoConfigure[IO]) // register AWS Lambda detector .addResourceDetector(AWSLambdaDetector[IO]) + // register AWS EC2 detector + .addResourceDetector(AWSEC2Detector[IO]) ) .use { autoConfigured => program(autoConfigured.tracerProvider) @@ -166,8 +231,8 @@ There are several ways to configure the options: Add settings to the `build.sbt`: ```scala -javaOptions += "-Dotel.otel4s.resource.detectors.enabled=aws-lambda" -envVars ++= Map("OTEL_OTEL4S_RESOURCE_DETECTORS_ENABLE" -> "aws-lambda") +javaOptions += "-Dotel.otel4s.resource.detectors.enabled=aws-lambda,aws-ec2" +envVars ++= Map("OTEL_OTEL4S_RESOURCE_DETECTORS_ENABLE" -> "aws-lambda,aws-ec2") ``` @:choice(scala-cli) @@ -175,12 +240,12 @@ envVars ++= Map("OTEL_OTEL4S_RESOURCE_DETECTORS_ENABLE" -> "aws-lambda") Add directives to the `*.scala` file: ```scala -//> using javaOpt -Dotel.otel4s.resource.detectors.enabled=aws-lambda +//> using javaOpt -Dotel.otel4s.resource.detectors.enabled=aws-lambda,aws-ec2 ``` @:choice(shell) ```shell -$ export OTEL_OTEL4S_RESOURCE_DETECTORS_ENABLED=aws-lambda +$ export OTEL_OTEL4S_RESOURCE_DETECTORS_ENABLED=aws-lambda,aws-ec2 ``` @:@ diff --git a/sdk-contrib/aws/resource/src/main/scala/org/typelevel/otel4s/sdk/contrib/aws/resource/AWSEC2Detector.scala b/sdk-contrib/aws/resource/src/main/scala/org/typelevel/otel4s/sdk/contrib/aws/resource/AWSEC2Detector.scala new file mode 100644 index 000000000..c546c7885 --- /dev/null +++ b/sdk-contrib/aws/resource/src/main/scala/org/typelevel/otel4s/sdk/contrib/aws/resource/AWSEC2Detector.scala @@ -0,0 +1,220 @@ +/* + * Copyright 2024 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.typelevel.otel4s.sdk.contrib.aws.resource + +import cats.effect.Async +import cats.effect.Resource +import cats.effect.std.Console +import cats.syntax.applicativeError._ +import cats.syntax.flatMap._ +import cats.syntax.functor._ +import fs2.io.net.Network +import io.circe.Decoder +import org.http4s.EntityDecoder +import org.http4s.Header +import org.http4s.Headers +import org.http4s.Method +import org.http4s.Request +import org.http4s.Uri +import org.http4s.circe._ +import org.http4s.client.Client +import org.http4s.ember.client.EmberClientBuilder +import org.http4s.syntax.literals._ +import org.typelevel.ci._ +import org.typelevel.otel4s.AttributeKey +import org.typelevel.otel4s.Attributes +import org.typelevel.otel4s.sdk.TelemetryResource +import org.typelevel.otel4s.sdk.resource.TelemetryResourceDetector +import org.typelevel.otel4s.semconv.SchemaUrls + +import scala.concurrent.duration._ + +private class AWSEC2Detector[F[_]: Async: Network: Console] private ( + baseUri: Uri, + customClient: Option[Client[F]] +) extends TelemetryResourceDetector[F] { + + import AWSEC2Detector.Const + import AWSEC2Detector.Keys + import AWSEC2Detector.IdentityMetadata + + def name: String = Const.Name + + def detect: F[Option[TelemetryResource]] = + mkClient.use { client => + retrieve(client).handleErrorWith { e => + Console[F] + .errorln(s"AWSEC2Detector: cannot retrieve metadata from $baseUri due to ${e.getMessage}") + .as(None) + } + } + + private def retrieve(client: Client[F]): F[Option[TelemetryResource]] = + for { + token <- retrieveToken(client) + metadata <- retrieveIdentityMetadata(client, token) + hostname <- retrieveHostname(client, token) + } yield Some(build(metadata, hostname)) + + private def retrieveIdentityMetadata(client: Client[F], token: String): F[IdentityMetadata] = { + implicit val decoder: EntityDecoder[F, IdentityMetadata] = accumulatingJsonOf[F, IdentityMetadata] + + val request = Request[F]( + Method.GET, + uri = baseUri / "latest" / "dynamic" / "instance-identity" / "document", + headers = Headers(Header.Raw(Const.TokenHeader, token)) + ) + + client.expect[IdentityMetadata](request) + } + + private def retrieveHostname(client: Client[F], token: String): F[String] = { + val request = Request[F]( + Method.GET, + uri = baseUri / "latest" / "meta-data" / "hostname", + headers = Headers(Header.Raw(Const.TokenHeader, token)) + ) + + client.expect[String](request) + } + + private def retrieveToken(client: Client[F]): F[String] = { + val request = Request[F]( + Method.PUT, + uri = baseUri / "latest" / "api" / "token", + headers = Headers(Header.Raw(Const.TokenTTLHeader, "60")) + ) + + client.expect[String](request) + } + + private def build(metadata: IdentityMetadata, hostname: String): TelemetryResource = { + val builder = Attributes.newBuilder + + builder.addOne(Keys.CloudProvider, Const.CloudProvider) + builder.addOne(Keys.CloudPlatform, Const.CloudPlatform) + builder.addOne(Keys.CloudAccountId, metadata.accountId) + builder.addOne(Keys.CloudRegion, metadata.region) + builder.addOne(Keys.CloudAvailabilityZones, metadata.availabilityZone) + builder.addOne(Keys.HostId, metadata.instanceId) + builder.addOne(Keys.HostType, metadata.instanceType) + builder.addOne(Keys.HostImageId, metadata.imageId) + builder.addOne(Keys.HostName, hostname) + + TelemetryResource(builder.result(), Some(SchemaUrls.Current)) + } + + private def mkClient: Resource[F, Client[F]] = + customClient match { + case Some(client) => Resource.pure(client) + case None => EmberClientBuilder.default[F].withTimeout(Const.Timeout).build + } + +} + +object AWSEC2Detector { + + private object Const { + val Name = "aws-ec2" + val CloudProvider = "aws" + val CloudPlatform = "aws_ec2" + val MetadataEndpoint = uri"http://169.254.169.254" + val Timeout: FiniteDuration = 2.seconds + val TokenHeader = ci"X-aws-ec2-metadata-token" + val TokenTTLHeader = ci"X-aws-ec2-metadata-token-ttl-seconds" + } + + private object Keys { + val CloudProvider: AttributeKey[String] = AttributeKey("cloud.provider") + val CloudPlatform: AttributeKey[String] = AttributeKey("cloud.platform") + val CloudAccountId: AttributeKey[String] = AttributeKey("cloud.account.id") + val CloudAvailabilityZones: AttributeKey[String] = AttributeKey("cloud.availability_zone") + val CloudRegion: AttributeKey[String] = AttributeKey("cloud.region") + val HostId: AttributeKey[String] = AttributeKey("host.id") + val HostType: AttributeKey[String] = AttributeKey("host.type") + val HostImageId: AttributeKey[String] = AttributeKey("host.image.id") + val HostName: AttributeKey[String] = AttributeKey("host.name") + } + + private final case class IdentityMetadata( + accountId: String, + region: String, + availabilityZone: String, + instanceId: String, + instanceType: String, + imageId: String + ) + + private object IdentityMetadata { + implicit val metadataDecoder: Decoder[IdentityMetadata] = + Decoder.forProduct6( + "accountId", + "region", + "availabilityZone", + "instanceId", + "instanceType", + "imageId" + )(IdentityMetadata.apply) + } + + /** The detector fetches instance metadata from the `http://169.254.169.254` endpoint. + * + * @example + * {{{ + * OpenTelemetrySdk + * .autoConfigured[IO]( + * // register OTLP exporters configurer + * _.addExportersConfigurer(OtlpExportersAutoConfigure[IO]) + * // register AWS EC2 detector + * .addResourceDetector(AWSEC2Detector[IO]) + * ) + * .use { autoConfigured => + * val sdk = autoConfigured.sdk + * ??? + * } + * }}} + * + * @see + * [[https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html]] + */ + def apply[F[_]: Async: Network: Console]: TelemetryResourceDetector[F] = + new AWSEC2Detector[F](Const.MetadataEndpoint, None) + + /** The detector fetches instance metadata from the given `baseUri` using the given `client`. + * + * @example + * {{{ + * OpenTelemetrySdk + * .autoConfigured[IO]( + * // register OTLP exporters configurer + * _.addExportersConfigurer(OtlpExportersAutoConfigure[IO]) + * // register AWS EC2 detector + * .addResourceDetector(AWSEC2Detector[IO]) + * ) + * .use { autoConfigured => + * val sdk = autoConfigured.sdk + * ??? + * } + * }}} + * + * @see + * [[https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html]] + */ + def apply[F[_]: Async: Network: Console](baseUri: Uri, client: Client[F]): TelemetryResourceDetector[F] = + new AWSEC2Detector[F](baseUri, Some(client)) + +} diff --git a/sdk-contrib/aws/resource/src/test/scala/org/typelevel/otel4s/sdk/contrib/aws/resource/AWSEC2DetectorSuite.scala b/sdk-contrib/aws/resource/src/test/scala/org/typelevel/otel4s/sdk/contrib/aws/resource/AWSEC2DetectorSuite.scala new file mode 100644 index 000000000..c464e00f0 --- /dev/null +++ b/sdk-contrib/aws/resource/src/test/scala/org/typelevel/otel4s/sdk/contrib/aws/resource/AWSEC2DetectorSuite.scala @@ -0,0 +1,115 @@ +/* + * Copyright 2024 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.typelevel.otel4s.sdk.contrib.aws.resource + +import cats.effect.IO +import cats.syntax.functor._ +import io.circe.Json +import io.circe.syntax._ +import munit.CatsEffectSuite +import org.http4s.HttpApp +import org.http4s.HttpRoutes +import org.http4s.Request +import org.http4s.client.Client +import org.http4s.dsl.io._ +import org.http4s.syntax.literals._ +import org.typelevel.ci._ +import org.typelevel.otel4s.Attributes +import org.typelevel.otel4s.sdk.TelemetryResource +import org.typelevel.otel4s.semconv.SchemaUrls +import org.typelevel.otel4s.semconv.experimental.attributes.CloudExperimentalAttributes._ +import org.typelevel.otel4s.semconv.experimental.attributes.HostExperimentalAttributes._ + +class AWSEC2DetectorSuite extends CatsEffectSuite { + + test("parse metadata response and add attributes") { + val accountId = "1234567890" + val availabilityZone = "eu-west-1a" + val region = "eu-west-1" + val instanceId = "i-abc321de" + val instanceType = "t3.small" + val imageId = "ami-abc123de" + val hostname = "ip-10-0-0-1.eu-west-1.compute.internal" + + val metadata = Json.obj( + "accountId" := accountId, + "architecture" := "x86_64", + "availabilityZone" := availabilityZone, + "imageId" := imageId, + "instanceId" := instanceId, + "instanceType" := instanceType, + "privateIp" := "10.0.0.1", + "region" := region, + "version" := "2017-09-30" + ) + + val client = Client.fromHttpApp(mockServer(metadata, hostname, "token")) + + val expected = TelemetryResource( + Attributes( + CloudProvider(CloudProviderValue.Aws.value), + CloudPlatform(CloudPlatformValue.AwsEc2.value), + CloudAccountId(accountId), + CloudRegion(region), + CloudAvailabilityZone(availabilityZone), + HostId(instanceId), + HostType(instanceType), + HostImageId(imageId), + HostName(hostname) + ), + Some(SchemaUrls.Current) + ) + + AWSEC2Detector[IO](uri"", client).detect.assertEquals(Some(expected)) + } + + test("return None when metadata response is unparsable") { + val client = Client.fromHttpApp(mockServer(Json.obj(), "", "")) + AWSEC2Detector[IO](uri"", client).detect.assertEquals(None) + } + + test("return None when the endpoint is unavailable exist") { + val client = Client.fromHttpApp(HttpApp.notFound[IO]) + AWSEC2Detector[IO](uri"", client).detect.assertEquals(None) + } + + private def mockServer(metadata: Json, hostname: String, token: String): HttpApp[IO] = { + def checkHeader(req: Request[IO]): Either[Throwable, Unit] = + req.headers + .get(ci"X-aws-ec2-metadata-token") + .toRight(new RuntimeException("Token header is missing")) + .flatMap { header => + Either.cond(header.head.value == token, (), new RuntimeException("Invalid token header value")) + } + .void + + HttpRoutes + .of[IO] { + case req @ GET -> Root / "latest" / "meta-data" / "hostname" => + IO.fromEither(checkHeader(req)) >> Ok(hostname) + + case req @ GET -> Root / "latest" / "dynamic" / "instance-identity" / "document" => + import org.http4s.circe.jsonEncoder + IO.fromEither(checkHeader(req)) >> Ok(metadata) + + case PUT -> Root / "latest" / "api" / "token" => + Ok(token) + } + .orNotFound + } + +}