diff --git a/build.sbt b/build.sbt index ebbea32142..215c6150b8 100644 --- a/build.sbt +++ b/build.sbt @@ -62,6 +62,7 @@ lazy val V = new { val typesafeConfig = "1.4.2" val protobuf = "3.1.9" val testContainersScala = "0.41.0" + val testContainersJavaKeycloak = "3.0.0" val doobie = "1.0.0-RC2" val quill = "4.7.3" @@ -121,6 +122,7 @@ lazy val D = new { // TODO we are adding test stuff to the main dependencies val testcontainersPostgres: ModuleID = "com.dimafeng" %% "testcontainers-scala-postgresql" % V.testContainersScala val testcontainersVault: ModuleID = "com.dimafeng" %% "testcontainers-scala-vault" % V.testContainersScala + val testcontainersKeycloak: ModuleID = "com.github.dasniko" % "testcontainers-keycloak" % V.testContainersJavaKeycloak val doobiePostgres: ModuleID = "org.tpolecat" %% "doobie-postgres" % V.doobie val doobieHikari: ModuleID = "org.tpolecat" %% "doobie-hikari" % V.doobie @@ -149,6 +151,7 @@ lazy val D_Shared = new { D.scalaPbGrpc, D.testcontainersPostgres, D.testcontainersVault, + D.testcontainersKeycloak, D.zio, // FIXME: split shared DB stuff as subproject? D.doobieHikari, @@ -157,6 +160,26 @@ lazy val D_Shared = new { ) } +lazy val D_SharedTest = new { + lazy val dependencies: Seq[ModuleID] = + Seq( + D.typesafeConfig, + D.testcontainersPostgres, + D.testcontainersVault, + D.testcontainersKeycloak, + D.zio, + D.doobieHikari, + D.doobiePostgres, + D.zioCatsInterop, + D.zioJson, + D.zioHttp, + D.zioTest, + D.zioTestSbt, + D.zioTestMagnolia, + D.zioMock + ) +} + lazy val D_Connect = new { private lazy val logback = "ch.qos.logback" % "logback-classic" % V.logback % Test @@ -393,6 +416,19 @@ lazy val shared = (project in file("shared")) ) .enablePlugins(BuildInfoPlugin) +lazy val sharedTest = (project in file("shared-test")) + // .configure(publishConfigure) + .settings( + organization := "io.iohk.atala", + organizationName := "Input Output Global", + buildInfoPackage := "io.iohk.atala.sharedtest", + name := "sharedtest", + crossPaths := false, + libraryDependencies ++= D_SharedTest.dependencies + ) + .dependsOn(shared) + .enablePlugins(BuildInfoPlugin) + // ######################### // ### Models & Services ### // ######################### @@ -752,7 +788,9 @@ lazy val prismAgentWalletAPI = project .settings(prismAgentConnectCommonSettings) .settings( name := "prism-agent-wallet-api", - libraryDependencies ++= D_PrismAgent.keyManagementDependencies ++ D_PrismAgent.postgresDependencies ++ Seq(D.zioMock) + libraryDependencies ++= D_PrismAgent.keyManagementDependencies ++ D_PrismAgent.postgresDependencies ++ Seq( + D.zioMock + ) ) .dependsOn( agentDidcommx, @@ -807,6 +845,7 @@ releaseProcess := Seq[ReleaseStep]( lazy val aggregatedProjects: Seq[ProjectReference] = Seq( shared, + sharedTest, models, protocolConnection, protocolCoordinateMediation, diff --git a/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/KeycloakContainerCustom.scala b/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/KeycloakContainerCustom.scala new file mode 100644 index 0000000000..aefa20a621 --- /dev/null +++ b/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/KeycloakContainerCustom.scala @@ -0,0 +1,39 @@ +package io.iohk.atala.sharedtest.containers + +import com.dimafeng.testcontainers.SingleContainer +import dasniko.testcontainers.keycloak.ExtendableKeycloakContainer +import KeycloakTestContainer.keycloakContainer +import org.testcontainers.utility.DockerImageName +import zio.{TaskLayer, ZIO, ZLayer} + +final class KeycloakContainerCustom( + dockerImageNameOverride: DockerImageName, + isOnGithubRunner: Boolean = false +) extends SingleContainer[ExtendableKeycloakContainer[_]] { + + private val keycloakContainer: ExtendableKeycloakContainer[_] = new ExtendableKeycloakContainer( + dockerImageNameOverride.toString + ) { + override def getHost: String = { + if (isOnGithubRunner) super.getContainerId.take(12) + else super.getHost + } +// override def getMappedPort(originalPort: Int): Integer = { +// if (isOnGithubRunner) 8300 +// else super.getMappedPort(originalPort) +// } + } + + override val container: ExtendableKeycloakContainer[_] = keycloakContainer +} + +object KeycloakContainerCustom { + val layer: TaskLayer[KeycloakContainerCustom] = + ZLayer.scoped { + ZIO + .acquireRelease(ZIO.attemptBlockingIO { + keycloakContainer() + })(container => ZIO.attemptBlockingIO(container.stop()).orDie) + .tap(container => ZIO.attemptBlocking(container.start())) + } +} diff --git a/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/KeycloakTestContainer.scala b/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/KeycloakTestContainer.scala new file mode 100644 index 0000000000..23badf2508 --- /dev/null +++ b/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/KeycloakTestContainer.scala @@ -0,0 +1,23 @@ +package io.iohk.atala.sharedtest.containers + +import org.testcontainers.containers.output.OutputFrame +import org.testcontainers.utility.DockerImageName + +object KeycloakTestContainer { + def keycloakContainer( + imageName: String = "quay.io/keycloak/keycloak:22.0.4", + ): KeycloakContainerCustom = { + val isOnGithubRunner = sys.env.contains("GITHUB_NETWORK") + val container = + new KeycloakContainerCustom( + dockerImageNameOverride = DockerImageName.parse(imageName), + isOnGithubRunner = isOnGithubRunner + ) + + sys.env.get("GITHUB_NETWORK").map { network => + container.container.withNetworkMode(network) + } + + container + } +} diff --git a/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/KeycloakTestContainerSupport.scala b/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/KeycloakTestContainerSupport.scala new file mode 100644 index 0000000000..c29d5fecc7 --- /dev/null +++ b/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/KeycloakTestContainerSupport.scala @@ -0,0 +1,13 @@ +package io.iohk.atala.sharedtest.containers +import org.keycloak.admin.client.Keycloak +import zio.* + +type KeycloakAdminClient = Keycloak + +trait KeycloakTestContainerSupport { + protected val keycloakContainerLayer: TaskLayer[KeycloakContainerCustom] = + KeycloakContainerCustom.layer + + protected val keycloakAdminClientLayer: URLayer[KeycloakContainerCustom, KeycloakAdminClient] = + ZLayer.fromZIO(ZIO.service[KeycloakContainerCustom].map(_.container.getKeycloakAdminClient)) +} diff --git a/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/PostgreSQLContainerCustom.scala b/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/PostgreSQLContainerCustom.scala new file mode 100644 index 0000000000..b77d3094e6 --- /dev/null +++ b/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/PostgreSQLContainerCustom.scala @@ -0,0 +1,51 @@ +package io.iohk.atala.sharedtest.containers + +import com.dimafeng.testcontainers.{JdbcDatabaseContainer, PostgreSQLContainer} +import org.testcontainers.utility.DockerImageName + +class PostgreSQLContainerCustom( + dockerImageNameOverride: Option[DockerImageName] = None, + databaseName: Option[String] = None, + pgUsername: Option[String] = None, + pgPassword: Option[String] = None, + mountPostgresDataToTmpfs: Boolean = false, + urlParams: Map[String, String] = Map.empty, + commonJdbcParams: JdbcDatabaseContainer.CommonParams = JdbcDatabaseContainer.CommonParams() +) extends PostgreSQLContainer( + dockerImageNameOverride, + databaseName, + pgUsername, + pgPassword, + mountPostgresDataToTmpfs, + urlParams, + commonJdbcParams + ) { + + override def jdbcUrl: String = { + /* This is such a hack! + * + * We are running PostgreSQL test containers inside a bridged (user-derfined) + * network. Testcontainers expects to be able to connect to the _host_ and + * map ports on the host. However we are running _inside_ a docker container. + * So now the mapping to _localhost:randomport_ -> spawned postgres:5432 is + * available from _outside_, but not form the docker container actually + * spawning the others. + * + * We also can't refer to them by name, because docker somehow fails to + * resolve names sometimes once a container has joined a network but didn't + * get a name assigned when joining :shurg:. + * + * We can however refer to containers by their containerId, or more + * precisely by their _short_ (first 12 char) Id. + * + * So we overwrite the jdbcUrl, and change the way it's constructed in test + * containers. + * + * This is a mess :( + */ + val origUrl = super.jdbcUrl + val idx = origUrl.indexOf('?') + val params = if (idx >= 0) origUrl.substring(idx) else "" + s"jdbc:postgresql://${containerId.take(12)}:5432/${super.databaseName}${params}" + } +} diff --git a/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/PostgresLayer.scala b/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/PostgresLayer.scala new file mode 100644 index 0000000000..4cd5dcfba2 --- /dev/null +++ b/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/PostgresLayer.scala @@ -0,0 +1,39 @@ +package io.iohk.atala.sharedtest.containers + +import com.dimafeng.testcontainers.PostgreSQLContainer +import doobie.util.transactor.Transactor +import io.iohk.atala.shared.db.ContextAwareTask +import io.iohk.atala.shared.db.DbConfig +import io.iohk.atala.shared.db.TransactorLayer +import io.iohk.atala.shared.test.containers.PostgresTestContainer.postgresContainer +import zio.* + +object PostgresLayer { + + def postgresLayer( + imageName: Option[String] = Some("postgres:13"), + verbose: Boolean = false + ): TaskLayer[PostgreSQLContainer] = + ZLayer.scoped { + ZIO + .acquireRelease(ZIO.attemptBlockingIO { + postgresContainer(imageName, verbose) + })(container => ZIO.attemptBlockingIO(container.stop()).orDie) + // Start the container outside the aquireRelease as this might fail + // to ensure contianer.stop() is added to the finalizer + .tap(container => ZIO.attemptBlocking(container.start())) + } + + private def dbConfig(container: PostgreSQLContainer): DbConfig = { + DbConfig( + username = container.username, + password = container.password, + jdbcUrl = container.jdbcUrl + ) + } + + lazy val dbConfigLayer: ZLayer[PostgreSQLContainer, Nothing, DbConfig] = + ZLayer.fromZIO { ZIO.serviceWith[PostgreSQLContainer](dbConfig) } + + def transactor: ZLayer[DbConfig, Throwable, Transactor[ContextAwareTask]] = TransactorLayer.contextAwareTask +} diff --git a/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/PostgresTestContainer.scala b/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/PostgresTestContainer.scala new file mode 100644 index 0000000000..546b44804f --- /dev/null +++ b/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/PostgresTestContainer.scala @@ -0,0 +1,27 @@ +package io.iohk.atala.sharedtest.containers + +import com.dimafeng.testcontainers.PostgreSQLContainer +import org.testcontainers.containers.output.OutputFrame +import org.testcontainers.utility.DockerImageName + +object PostgresTestContainer { + def postgresContainer( + imageName: Option[String] = Some("postgres:13"), + verbose: Boolean = false + ): PostgreSQLContainer = { + val container = + if (sys.env.contains("GITHUB_NETWORK")) + new PostgreSQLContainerCustom(dockerImageNameOverride = imageName.map(DockerImageName.parse)) + else + new PostgreSQLContainer(dockerImageNameOverride = imageName.map(DockerImageName.parse)) + sys.env.get("GITHUB_NETWORK").map { network => + container.container.withNetworkMode(network) + } + if (verbose) { + container.container + .withLogConsumer((t: OutputFrame) => println(t.getUtf8String)) + .withCommand("postgres", "-c", "log_statement=all", "-c", "log_destination=stderr") + } + container + } +} diff --git a/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/PostgresTestContainerSupport.scala b/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/PostgresTestContainerSupport.scala new file mode 100644 index 0000000000..2527765319 --- /dev/null +++ b/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/PostgresTestContainerSupport.scala @@ -0,0 +1,64 @@ +package io.iohk.atala.sharedtest.containers + +import com.dimafeng.testcontainers.PostgreSQLContainer +import doobie.util.transactor.Transactor +import io.iohk.atala.shared.db.ContextAwareTask +import io.iohk.atala.shared.db.TransactorLayer +import zio.* + +trait PostgresTestContainerSupport { + + protected val pgContainerLayer: TaskLayer[PostgreSQLContainer] = PostgresLayer.postgresLayer() + + protected val contextAwareTransactorLayer: TaskLayer[Transactor[ContextAwareTask]] = { + import doobie.* + import doobie.implicits.* + import zio.interop.catz.* + + val appUser = "test-application-user" + val appPassword = "password" + + val createAppUser = (xa: Transactor[Task]) => + doobie.free.connection.createStatement + .map { stm => + stm.execute(s"""CREATE USER "$appUser" WITH PASSWORD '$appPassword';""") + stm.execute( + s"""ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO "$appUser";""" + ) + stm + } + .transact(xa) + .unit + + val superUserTransactor = ZLayer.makeSome[PostgreSQLContainer, Transactor[Task]]( + TransactorLayer.task, + PostgresLayer.dbConfigLayer, + ) + + val appUserTransactor = ZLayer.makeSome[PostgreSQLContainer, Transactor[ContextAwareTask]]( + TransactorLayer.contextAwareTask, + PostgresLayer.dbConfigLayer.map(conf => + ZEnvironment( + conf.get.copy( + username = appUser, + password = appPassword + ) + ) + ), + ) + + val initializedTransactor = ZLayer.fromZIO { + for { + _ <- ZIO + .serviceWithZIO[Transactor[Task]](createAppUser) + .provideSomeLayer(superUserTransactor) + } yield appUserTransactor + }.flatten + + pgContainerLayer >>> initializedTransactor + } + + protected val systemTransactorLayer: TaskLayer[Transactor[Task]] = { + pgContainerLayer >>> PostgresLayer.dbConfigLayer >>> TransactorLayer.task + } +} diff --git a/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/VaultContainerCustom.scala b/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/VaultContainerCustom.scala new file mode 100644 index 0000000000..3500076bfd --- /dev/null +++ b/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/VaultContainerCustom.scala @@ -0,0 +1,32 @@ +package io.iohk.atala.sharedtest.containers + +import com.dimafeng.testcontainers.{SingleContainer, VaultContainer} +import org.testcontainers.vault.{VaultContainer => JavaVaultContainer} +import org.testcontainers.utility.DockerImageName + +/** See PostgreSQLContainerCustom for explanation */ +class VaultContainerCustom( + dockerImageNameOverride: DockerImageName, + vaultToken: Option[String] = None, + secrets: Option[VaultContainer.Secrets] = None, + isOnGithubRunner: Boolean = false +) extends SingleContainer[JavaVaultContainer[_]] { + + private val vaultContainer: JavaVaultContainer[_] = new JavaVaultContainer(dockerImageNameOverride) { + override def getHost: String = { + if (isOnGithubRunner) super.getContainerId().take(12) + else super.getHost() + } + override def getMappedPort(originalPort: Int): Integer = { + if (isOnGithubRunner) 8200 + else super.getMappedPort(originalPort) + } + } + + if (vaultToken.isDefined) vaultContainer.withVaultToken(vaultToken.get) + secrets.foreach { x => + vaultContainer.withSecretInVault(x.path, x.firstSecret, x.secrets: _*) + } + + override val container: JavaVaultContainer[_] = vaultContainer +} diff --git a/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/VaultTestContainer.scala b/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/VaultTestContainer.scala new file mode 100644 index 0000000000..afb8dcdd7b --- /dev/null +++ b/shared-test/src/main/scala/io/iohk/atala/sharedtest/containers/VaultTestContainer.scala @@ -0,0 +1,28 @@ +package io.iohk.atala.sharedtest.containers + +import org.testcontainers.containers.output.OutputFrame +import org.testcontainers.utility.DockerImageName + +object VaultTestContainer { + def vaultContainer( + imageName: String = "hashicorp/vault:1.15.0", + vaultToken: Option[String] = None, + verbose: Boolean = false + ): VaultContainerCustom = { + val isOnGithubRunner = sys.env.contains("GITHUB_NETWORK") + val container = + new VaultContainerCustom( + dockerImageNameOverride = DockerImageName.parse(imageName), + vaultToken = vaultToken, + isOnGithubRunner = isOnGithubRunner + ) + sys.env.get("GITHUB_NETWORK").map { network => + container.container.withNetworkMode(network) + } + if (verbose) { + container.container + .withLogConsumer((t: OutputFrame) => println(t.getUtf8String)) + } + container + } +} diff --git a/shared-test/src/test/scala/io/iohk/atala/sharedtest/containers/KeycloakTestContainerSupportSpec.scala b/shared-test/src/test/scala/io/iohk/atala/sharedtest/containers/KeycloakTestContainerSupportSpec.scala new file mode 100644 index 0000000000..c0c2fc10ac --- /dev/null +++ b/shared-test/src/test/scala/io/iohk/atala/sharedtest/containers/KeycloakTestContainerSupportSpec.scala @@ -0,0 +1,23 @@ +package io.iohk.atala.sharedtest.containers + +import zio.test.TestAspect.* +import zio.test.* +import zio.{Scope, ZIO} + +import scala.util.Try +object KeycloakTestContainerSupportSpec extends ZIOSpecDefault with KeycloakTestContainerSupport { + + override def spec = suite("KeycloakTestContainerSupportSpec")( + test("Keycloak container should be started") { + for { + keycloakContainer <- ZIO.service[KeycloakContainerCustom] + } yield assertTrue(keycloakContainer.container.isRunning) + }, + test("Keycloak admin-client works") { + for { + adminClient <- ZIO.service[KeycloakAdminClient] + usersCount <- ZIO.fromTry(Try(adminClient.realm("master").users().count())) + } yield assertCompletes + } + ).provideLayerShared(keycloakContainerLayer >+> keycloakAdminClientLayer) @@ sequential +}