Number of regression fixes and sbt/mill improvements (#308)

* better resoulution of mill scalacOptions based on scalaVersion

* add defualt repo to run.sh script

* Better injection of custom repositories

* Handle failure in evaluator when mapping scalac options

* Add global resolvers to sbt to fix epfl-lara/lisa

* sbt: Fix compilation in some version of sbt

* sbt: For sbt 1.9.x only when JDK 21 is used, revert minimal version to 1.6.2

* config: Disable tests in frequently failing projects

* scripts: Short summary of status in ./run.sh

* versioing: set version to v0.3.6

* mill: Try fix issues found in CI

* mill: Try fix resolving directory and supress warning

.github/actions/build-project/action.yaml

@@ -110,7 +110,7 @@ runs:
       uses: addnab/docker-run-action@v3
       if: steps.check-history.outputs.can-skip-build != 'true'
       with:
-        image: "virtuslab/scala-community-build-project-builder:jdk${{ env.java-version }}-v0.3.5"
+        image: "virtuslab/scala-community-build-project-builder:jdk${{ env.java-version }}-v0.3.6"
         options: -v ${{ github.workspace }}:/opencb/ -v ${{ github.workspace }}/github_key:/root/.ssh/github_key:ro
         run: |
           # Setup ssh required for downloading submodules

.github/actions/setup-build/action.yaml

@@ -75,7 +75,7 @@ runs:
       uses: addnab/docker-run-action@v3
       if: steps.check-published.outputs.is-compiler-published == 'false'
       with:
-        image: "virtuslab/scala-community-build-compiler-builder:v0.3.5"
+        image: "virtuslab/scala-community-build-compiler-builder:v0.3.6"
         options: -v ${{ github.workspace }}/compiler:/compiler/
         run: |
           Version="${{ steps.calc-version.outputs.effective-scala-version }}"

cli/scb-cli.scala

@@ -31,7 +31,7 @@ class FailedProjectException(msg: String)
     with NoStackTrace
 
 val communityBuildVersion =
-  sys.props.getOrElse("communitybuild.version", "v0.3.5")
+  sys.props.getOrElse("communitybuild.version", "v0.3.6")
 private val CBRepoName = "VirtusLab/community-build3"
 val projectBuilderUrl =
   s"https://raw.githubusercontent.com/$CBRepoName/master/project-builder"

coordinator/configs/projects-config.conf

@@ -412,6 +412,7 @@ jchapuis_fs2-kafka-mock{
     "removeScalacOptionsStartingWith -P:wartremover"
   ]
 }
+jcouyang_dhall-generic.sbt.commands = ["set every Test/classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat"]
 jmcardon_tsec.tests=compile-only
 joan38_kubernetes-client.tests = compile-only
 jobial-io_scase{
@@ -640,6 +641,7 @@ pureharm_pureharm-db-core{
     "com.busymachines%pureharm-db-testkit"
   ]
 }
+purplekingdomgames_ultraviolet.tests = compile-only
 
 ranyitz_brush.tests = compile-only
 rediscala_rediscala.tests = compile-only
@@ -657,6 +659,7 @@ reactivemongo_reactivemongo-bson {
   projects.exclude = ["reactivemongo-jmx", "reactivemongo-datadog", "reactivemongo-kamon"]
 }
 reactivemongo_reactivemongo-play-json.tests = compile-only // fails due to changes in reported error messages
+rescala-lang_rescala.tests = compile-only
 ruimo_csvparser.tests = disabled // does not compile
 ruimo_graphics.tests = disabled // does not compile
 ruimo_scoins.tests = disabled // does not compile
@@ -700,12 +703,7 @@ scalatest_scalatestplus-scalacheck { sbt.commands = ["set Compile/doc/skip := tr
 scalatest_scalatestplus-selenium   { sbt.commands = ["set Compile/doc/skip := true"] }
 scalatest_scalatestplus-scalacheck { sbt.commands = ["set scalatestPlusScalaCheckJVM/Compile/doc/skip := true"] }
 scala-native_scala-native-cli.tests = compile-only
-scalanlp_breeze {
-  sbt.commands=[
-    // flaky tests
-    """set math/Test/unmanagedSources/excludeFilter := HiddenFileFilter || "OptimizationSpaceTest.scala" || "LinearAlgebraTest.scala" || "DenseMatrixTest.scala" || "CSCMatrixTest.scala" || "LUTest.scala" || "ProjectedQuasiNewtonTest.scala" || "InvGammaTest.scala" """
-  ]
-}
+scalanlp_breeze.tests = compile-only
 // Upstream is compiled with Java 11
 scalapb_scalapb {
   tests = compile-only // disabled due to usage of scalaOutputVersion, breaks due to changes in LazyVals
@@ -963,12 +961,14 @@ vagmcs_optimus{
   tests = compile-only // Fails locally
 }
 vigoo_desert.java.version=17
+vigoo_prox.tests = compile-only 
 virtuslab_scala-cli {
   java.version=17
   projects.overrides{
     build-module.tests = compile-only
     build-macros.tests = compile-only
     cli.tests = compile-only
+    directives.tests = compile-only // uses Scala 3.0.2 runner
   }
 }
 vladkopanev_cats-saga.source-patches = [

project-builder/build-revision.sh

@@ -96,6 +96,7 @@ fi
 
 echo ""
 echo "----"
+echo "started" > build-status.txt 
 if [ -f "repo/mill" ] || [ -f "repo/build.sc" ]; then
   echo "Mill project found: ${isMillProject}"
   echo "mill" > $buildToolFile

project-builder/mill/MillCommunityBuild.sc

@@ -51,8 +51,8 @@ import requests._
 import coursier.maven.MavenRepository
 import coursier.Repository
 
-trait CommunityBuildCoursierModule extends CoursierModule { self: JavaModule =>
-  protected val mavenRepoUrl: Option[String] = sys.props
+trait CommunityBuildCoursierModule extends CoursierModule with JavaModule {
+  private val mavenRepoUrl: Option[String] = sys.props
     .get("communitybuild.maven.url")
     .map(_.stripSuffix("/"))
   private val mavenRepo = mavenRepoUrl.map(MavenRepository(_))
@@ -63,7 +63,7 @@ trait CommunityBuildCoursierModule extends CoursierModule { self: JavaModule =>
   // Override zinc worker, we need to set custom repostitories there are well,
   // to allow to use our custom repo
   override def zincWorker = toZincWorker(CommunityBuildZincWorker)
-  object CommunityBuildZincWorker extends ZincWorkerModule with CoursierModule {
+  private object CommunityBuildZincWorker extends ZincWorkerModule with CoursierModule {
     override def repositoriesTask = T.task {
       mavenRepo.foldLeft(super.repositoriesTask())(_ :+ _)
     }
@@ -128,12 +128,12 @@ private object ScalacOptionsSettings {
   val remove = parse("communitybuild.removeScalacOptions")
 }
 
-def mapScalacOptions(current: Seq[String]): Seq[String] =
-  CommunityBuildCore.Scala3CommunityBuild.Utils.mapScalacOptions(
+def mapScalacOptions(scalaVersion: String, current: Seq[String]): Seq[String] = 
+  if(scalaVersion.startsWith("3.")) CommunityBuildCore.Scala3CommunityBuild.Utils.mapScalacOptions(
     current = current,
     append = ScalacOptionsSettings.append,
     remove = ScalacOptionsSettings.remove
-  )
+  ) else current
 
 case class ModuleInfo(org: String, name: String, module: Module) {
   val targetName = s"$org%$name"
@@ -218,7 +218,7 @@ def runBuild(configJson: String, targets: Seq[String])(implicit ctx: Ctx) = {
       topLevelModules
     ).toList
   } yield {
-    ctx.log.info(s"Starting build for $name")
+    ctx.log.info(s"\nStarting build for $name")
     val evaluator = new MillTaskEvaluator()
     import evaluator._
     val overrides = {

project-builder/mill/prepare-project.sh

@@ -64,7 +64,6 @@ if [[ "$millBinaryVersionMajor" -ne "0" || ( "$millBinaryVersionMinor" -lt "9" |
   echo "Unsupported mill version"
   exit 1
 fi
-ln $scriptDir/compat/$millBinaryVersion.sc MillVersionCompat.sc
 
 # Base64 is used to mitigate spliting json by whitespaces
 for elem in $(echo "${projectConfig}" | jq -r '.sourcePatches // [] | .[] | @base64'); do
@@ -100,17 +99,31 @@ fi
 
 # Rename build.sc to build.scala - Scalafix does ignore .sc files
 # Use scala 3 dialect to allow for top level defs
-cp build.sc build.scala \
-  && scalafix \
+adaptedFiles=($PWD/build.sc )
+if [[ -d ./project ]]; then
+  adaptedFiles+=(`find ./project -type f -name "*.sc"`)
+fi
+for scFile in "${adaptedFiles[@]}"; do 
+  echo "Apply scalafix rules to $scFile"
+  scalaFile="${scFile%.sc}.scala"
+  cp $scFile $scalaFile
+  scalafix \
     --rules file:${scriptDir}/scalafix/rules/src/main/scala/fix/Scala3CommunityBuildMillAdapter.scala \
-    --files build.scala \
+    --files $scalaFile \
     --stdout \
     --syntactic \
     --settings.Scala3CommunityBuildMillAdapter.targetScalaVersion "$scalaVersion" \
     --settings.Scala3CommunityBuildMillAdapter.targetPublishVersion "$publishVersion" \
     --settings.Scala3CommunityBuildMillAdapter.millBinaryVersion "$millBinaryVersion" \
-    --scala-version 3.1.0 > build.sc \
-  && rm build.scala 
+    --scala-version 3.1.0 > ${scFile}.adapted \
+    && mv ${scFile}.adapted $scFile \
+    || (echo "Failed to adapted $scFile, ignoring changes"; cat ${scFile}.adapted; rm -f ${scFile}.adapted) 
+  rm $scalaFile
+done
 
-ln -fs $scriptDir/../shared/CommunityBuildCore.scala CommunityBuildCore.sc
-ln -fs $scriptDir/MillCommunityBuild.sc MillCommunityBuild.sc
+for f in "${adaptedFiles[@]}"; do
+  dir="$(dirname $(realpath "$f"))"
+  ln -fs $scriptDir/../shared/CommunityBuildCore.scala ${dir}/CommunityBuildCore.sc
+  ln -fs $scriptDir/MillCommunityBuild.sc ${dir}/MillCommunityBuild.sc
+  ln -fs $scriptDir/compat/$millBinaryVersion.sc ${dir}/MillVersionCompat.sc
+done

project-builder/mill/scalafix/input/src/main/scala/fix/MillScalacOptionsOverride.scala

@@ -6,13 +6,14 @@ package fix
 object MillScalacOptionsOverride {
   object MillCommunityBuild{
     implicit class MillCommunityBuildScalacOptionsOps(asSeq: Seq[String]){
-      def mapScalacOptions(): Seq[String] = ???
+      def mapScalacOptions(scalaVersion: String): Seq[String] = ???
     }
     implicit class MillCommunityBuildScalacOptionsTargetOps(asTarget: mill.T[Seq[String]]){
-      def mapScalacOptions(): mill.T[Seq[String]] = ???
+      def mapScalacOptions(scalaVersion: String): mill.T[Seq[String]] = ???
     }
   }
   import MillCommunityBuild._
+  def scalaVersion: String = ???
   def scalacOptions = List.empty[String]
   object mill{
     import scala.language.implicitConversions

project-builder/mill/scalafix/output/src/main/scala/fix/MillScalacOptionsOverride.scala

@@ -3,14 +3,15 @@ package fix
 object MillScalacOptionsOverride {
   object MillCommunityBuild{
     implicit class MillCommunityBuildScalacOptionsOps(asSeq: Seq[String]){
-      def mapScalacOptions(): Seq[String] = ???
+      def mapScalacOptions(scalaVersion: String): Seq[String] = ???
     }
     implicit class MillCommunityBuildScalacOptionsTargetOps(asTarget: mill.T[Seq[String]]){
-      def mapScalacOptions(): mill.T[Seq[String]] = ???
+      def mapScalacOptions(scalaVersion: String): mill.T[Seq[String]] = ???
     }
   }
   import MillCommunityBuild._
-  def scalacOptions = { List.empty[String] }.mapScalacOptions()
+  def scalaVersion: String = ???
+  def scalacOptions = { List.empty[String] }.mapScalacOptions(scalaVersion)
   object mill{
     import scala.language.implicitConversions
     trait T[U]
@@ -21,29 +22,29 @@ object MillScalacOptionsOverride {
   import mill._
 
   object module {
-    val scalacOptions = { Nil }.mapScalacOptions()
+    val scalacOptions = { Nil }.mapScalacOptions(scalaVersion)
   }
   object module2 {
-    def scalacOptions: Seq[String] = { Seq("-Xprint:typer") }.mapScalacOptions()
+    def scalacOptions: Seq[String] = { Seq("-Xprint:typer") }.mapScalacOptions(scalaVersion)
   }
 
   object module3 {
-    def scalacOptions = { mill.T(module2.scalacOptions) }.mapScalacOptions()
+    def scalacOptions = { mill.T(module2.scalacOptions) }.mapScalacOptions(scalaVersion)
   }
 
   class moduleDef {
     def scalacOptions: T[Seq[String]] = { {
       val opt1 = "-release:11"
       Seq(opt1)
-    } }.mapScalacOptions()
+    } }.mapScalacOptions(scalaVersion)
   }
   class moduleDef2 extends moduleDef {
-    override val scalacOptions: T[Seq[String]] = { MillScalacOptionsOverride.scalacOptions }.mapScalacOptions()
+    override val scalacOptions: T[Seq[String]] = { MillScalacOptionsOverride.scalacOptions }.mapScalacOptions(scalaVersion)
   }
   object moduleDef3 extends moduleDef {
     override def scalacOptions = { T {
       super.scalacOptions ++ Nil
-    } }.mapScalacOptions()
+    } }.mapScalacOptions(scalaVersion)
   }
 
 }

project-builder/mill/scalafix/rules/src/main/scala/fix/Scala3CommunityBuildMillAdapter.scala

@@ -103,7 +103,7 @@ class Scala3CommunityBuildMillAdapter(
 
       case template @ Template(_, traits, _, _)
           if anyTreeOfTypeName(
-            has = Seq("ScalaModule", "JavaModule"),
+            has = coursierModuleSubtypes ++ testModuleSubtypes,
             butNot = Seq("PublishModule", "CoursierModule")
           )(traits) =>
         Patch.addRight(
@@ -130,7 +130,7 @@ class Scala3CommunityBuildMillAdapter(
         }
 
       case tree @ ValOrDefDef(Term.Name("scalacOptions"), _, body) =>
-        Patch.addAround(body,"{ ", " }.mapScalacOptions()")
+        Patch.addAround(body, "{ ", " }.mapScalacOptions(scalaVersion)")
 
       case ValOrDefDef(Term.Name(id), tpe, body) if scala3Identifiers.contains(id) =>
         body.toString().trim() match {
@@ -153,6 +153,29 @@ class Scala3CommunityBuildMillAdapter(
     headerInject + patch
   }
 
+  // format: off
+  val coursierModuleSubtypes = Seq(
+    "CrossModuleBase","CrossSbtModule","CrossSbtModuleTests","CrossScalaModule","CrossScalaVersionRanges",
+    "Giter8Module","Giter8Module",
+    "JavaModule","JavaModuleTests",
+    "MavenModule","MavenModuleTests",
+    "PlatformScalaModule","PublishModule",
+    "SbtModule","SbtModuleTests","SbtNativeModule","ScalaJSModule","ScalaJSTests","ScalaMetalsSupport","ScalaModule","ScalaNativeModule","ScalaNativeTests","ScalaTests","ScalafmtModule","ScalafmtModule","SemanticDbJavaModule",
+    "TestScalaJSModule","TestScalaNativeModule","Tests",
+    "UnidocModule",
+    "ZincWorkerModule"
+  )
+  val testModuleSubtypes = Seq(
+    "CrossSbtModuleTests",
+    "JavaModuleTests","Junit4","Junit5",
+    "MavenModuleTests","Munit",
+    "SbtModuleTests","ScalaJSTests","ScalaNativeTests","ScalaTest","ScalaTests","Specs2",
+    "TestNg","TestScalaJSModule","TestScalaNativeModule","Tests",
+    "Utest",
+    "Weaver",
+    "ZioTest",
+  )
+  // format: on
   object Replacment {
     val CommunityBuildCross = "MillCommunityBuildCross"
     val CommunityBuildPublishModule =
@@ -197,12 +220,22 @@ class Scala3CommunityBuildMillAdapter(
     val MapScalacOptionsOps = """
     |
     |implicit class MillCommunityBuildScalacOptionsOps(asSeq: Seq[String]){
-    |  def mapScalacOptions() = MillCommunityBuild.mapScalacOptions(asSeq)
+    |  def mapScalacOptions(scalaVersion: mill.define.Target[String])(implicit ctx: mill.api.Ctx): Seq[String] = {
+    |      try scalaVersion.evaluate(ctx).asSuccess.map(_.value)
+    |      catch { _: Throwable => None }
+    |    }.map(MillCommunityBuild.mapScalacOptions(_, asSeq))
+    |     .getOrElse {
+    |        println("Failed to resolve scalaVersion, assume it's Scala 3 project")
+    |        MillCommunityBuild.mapScalacOptions(sys.props.getOrElse("communitybuild.scala", "3.3.1"), asSeq)
+    |     }
+    |  def mapScalacOptions(scalaVersion: String) = MillCommunityBuild.mapScalacOptions(scalaVersion, asSeq)
     |}
+    |
     |implicit class MillCommunityBuildScalacOptionsTargetOps(asTarget: mill.define.Target[Seq[String]]){
-    |  def mapScalacOptions() = asTarget.map(MillCommunityBuild.mapScalacOptions(_))
+    |  def mapScalacOptions(scalaVersion: mill.define.Target[String]) = scalaVersion.zip(asTarget).map {
+    |    case (scalaVersion, scalacOptions) => MillCommunityBuild.mapScalacOptions(scalaVersion, scalacOptions)
+    |  }
     |}
-    |
     |""".stripMargin
 
     val injects = {

project-builder/sbt/CommunityBuildPlugin.scala

@@ -74,32 +74,28 @@ object CommunityBuildPlugin extends AutoPlugin {
 
   import complete.DefaultParsers._
 
-  def mvnRepoPublishSettings = sys.env
+  lazy val customMavenRepoRepository = sys.env
     .get("CB_MVN_REPO_URL")
     .filter(_.nonEmpty)
     .map("Community Build Repo".at(_))
-    .map { ourResolver =>
-      Seq(
-        externalResolvers := ourResolver +: externalResolvers.value
-      )
-    }
-    .getOrElse(Nil)
-
+    .toSeq
+
   override def projectSettings = Seq(
+    externalResolvers := customMavenRepoRepository ++ externalResolvers.value,
     // Fix for cyclic dependency when trying to use crossScalaVersion ~= ???
     crossScalaVersions := (thisProjectRef / crossScalaVersions).value,
     // Use explicitly required scala version, otherwise we might stumble onto the default projects Scala versions
     libraryDependencies ++= extraLibraryDependencies(
         buildScalaVersion = sys.props.get("communitybuild.scala").filter(_.nonEmpty),
         projectScalaVersion= (thisProjectRef / scalaVersion).value
       )
-  ) ++ mvnRepoPublishSettings
+  )
 
   private def extraLibraryDependencies(buildScalaVersion: Option[String], projectScalaVersion: String ) =
     if (!projectScalaVersion.startsWith("3.")) Nil
     else
-      Utils.extraLibraryDependencies(buildScalaVersion.getOrElse(projectScalaVersion)).map { 
-        case Utils.LibraryDependency(org, artifact, version, scalaCrossVersion) =>
+      Scala3CommunityBuild.Utils.extraLibraryDependencies(buildScalaVersion.getOrElse(projectScalaVersion)).map { 
+        case Scala3CommunityBuild.Utils.LibraryDependency(org, artifact, version, scalaCrossVersion) =>
           if(scalaCrossVersion) org %% artifact % version
           else org % artifact % version
       }
@@ -355,6 +351,7 @@ object CommunityBuildPlugin extends AutoPlugin {
   }
 
   override def globalSettings = Seq(
+    resolvers ++= customMavenRepoRepository,
     moduleMappings := { // Store settings in file to capture its original scala versions
       val moduleIds = mkMappings.value
       IO.write(