Skip to content

Commit

Permalink
Allow users to use a custom output dir via an env var (#3530)
Browse files Browse the repository at this point in the history
This allows users to change the Mill output directory (by default, `out`
under the project root) to a directory of their choice, via the
`MILL_OUTPUT_DIR` environment variable.

Fixes #3144
  • Loading branch information
alexarchambault authored Sep 26, 2024
1 parent 41ef503 commit 4c3d2cb
Show file tree
Hide file tree
Showing 14 changed files with 154 additions and 22 deletions.
4 changes: 4 additions & 0 deletions docs/modules/ROOT/pages/Out_Dir.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -111,3 +111,7 @@ This is very useful if Mill is being unexpectedly slow, and you want to find out

`mill-server/*`::
Each Mill server instance needs to keep some temporary files in one of these directories. Deleting it will also terminate the associated server instance, if it is still running.

== Using another location than the `out/` directory

include::example/depth/out-dir/1-custom-out.adoc[]
28 changes: 28 additions & 0 deletions example/depth/out-dir/1-custom-out/build.mill
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
// The default location for Mill's output directory is `out/` under the project workspace.
// A task `printDest` of a module `foo` will have a default scratch space folder
// `out/foo/printDest.dest/`:
package build

import mill._

object foo extends Module {
def printDest = Task {
println(T.dest)
}
}

/** Usage
> ./mill foo.printDest
...
.../out/foo/printDest.dest
*/

// If you'd rather use another location than `out/`, that lives
// in a faster or a writable filesystem for example, you can change the output directory
// via the `MILL_OUTPUT_DIR` environment variable.

/** Usage
> MILL_OUTPUT_DIR=build-stuff/working-dir ./mill foo.printDest
...
.../build-stuff/working-dir/foo/printDest.dest
*/
1 change: 1 addition & 0 deletions example/package.mill
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ object `package` extends RootModule with Module {
object modules extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "modules"))
object cross extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "cross"))
object large extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "large"))
object `out-dir` extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "out-dir"))
object sandbox extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "sandbox"))
object libraries extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "libraries"))
}
Expand Down
8 changes: 8 additions & 0 deletions integration/feature/output-directory/resources/build.mill
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package build

import mill._
import mill.scalalib._

object `package` extends RootModule with ScalaModule {
def scalaVersion = scala.util.Properties.versionNumberString
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package mill.integration

import mill.main.client.{EnvVars, OutFiles}
import mill.testkit.UtestIntegrationTestSuite
import utest._

object OutputDirectoryTests extends UtestIntegrationTestSuite {

def tests: Tests = Tests {
test("Output directory sanity check") - integrationTest { tester =>
import tester._
eval("__.compile").isSuccess ==> true
val defaultOutDir = workspacePath / OutFiles.defaultOut
assert(os.isDir(defaultOutDir))
}

test("Output directory elsewhere in workspace") - integrationTest { tester =>
import tester._
eval(
"__.compile",
env = millTestSuiteEnv + (EnvVars.MILL_OUTPUT_DIR -> "testing/test-out")
).isSuccess ==> true
val expectedOutDir = workspacePath / "testing/test-out"
val defaultOutDir = workspacePath / OutFiles.defaultOut
assert(os.isDir(expectedOutDir))
assert(!os.exists(defaultOutDir))
}

test("Output directory outside workspace") - integrationTest { tester =>
import tester._
val outDir = os.temp.dir() / "tmp-out"
eval(
"__.compile",
env = millTestSuiteEnv + (EnvVars.MILL_OUTPUT_DIR -> outDir.toString)
).isSuccess ==> true
val defaultOutDir = workspacePath / OutFiles.defaultOut
assert(os.isDir(outDir))
assert(!os.exists(defaultOutDir))
}
}
}
7 changes: 7 additions & 0 deletions main/client/src/mill/main/client/EnvVars.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,13 @@ public class EnvVars {

public static final String MILL_JVM_OPTS_PATH = "MILL_JVM_OPTS_PATH";


/**
* Output directory where Mill workers' state and Mill tasks output should be
* written to
*/
public static final String MILL_OUTPUT_DIR = "MILL_OUTPUT_DIR";

// INTERNAL ENVIRONMENT VARIABLES
/**
* Used to pass the Mill workspace root from the client to the server, so
Expand Down
11 changes: 10 additions & 1 deletion main/client/src/mill/main/client/OutFiles.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,19 @@
* and documentation about what they do
*/
public class OutFiles {

final private static String envOutOrNull = System.getenv(EnvVars.MILL_OUTPUT_DIR);

/**
* Default hard-coded value for the Mill `out/` folder path. Unless you know
* what you are doing, you should favor using [[out]] instead.
*/
final public static String defaultOut = "out";

/**
* Path of the Mill `out/` folder
*/
final public static String out = "out";
final public static String out = envOutOrNull == null ? defaultOut : envOutOrNull;

/**
* Path of the Mill "meta-build", used to compile the `build.sc` file so we can
Expand Down
12 changes: 9 additions & 3 deletions runner/src/mill/runner/CodeGen.scala
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ object CodeGen {
allScriptCode: Map[os.Path, String],
targetDest: os.Path,
enclosingClasspath: Seq[os.Path],
millTopLevelProjectRoot: os.Path
millTopLevelProjectRoot: os.Path,
output: os.Path
): Unit = {
for (scriptSource <- scriptSources) {
val scriptPath = scriptSource.path
Expand Down Expand Up @@ -94,6 +95,7 @@ object CodeGen {
projectRoot,
enclosingClasspath,
millTopLevelProjectRoot,
output,
scriptPath,
scriptFolderPath,
childAliases,
Expand All @@ -112,6 +114,7 @@ object CodeGen {
projectRoot: os.Path,
enclosingClasspath: Seq[os.Path],
millTopLevelProjectRoot: os.Path,
output: os.Path,
scriptPath: os.Path,
scriptFolderPath: os.Path,
childAliases: String,
Expand All @@ -126,7 +129,8 @@ object CodeGen {
segments,
scriptFolderPath,
enclosingClasspath,
millTopLevelProjectRoot
millTopLevelProjectRoot,
output
)

val instrument = new ObjectDataInstrument(scriptCode)
Expand Down Expand Up @@ -183,13 +187,15 @@ object CodeGen {
segments: Seq[String],
scriptFolderPath: os.Path,
enclosingClasspath: Seq[os.Path],
millTopLevelProjectRoot: os.Path
millTopLevelProjectRoot: os.Path,
output: os.Path
): String = {
s"""import _root_.mill.runner.MillBuildRootModule
|@_root_.scala.annotation.nowarn
|object MillMiscInfo extends MillBuildRootModule.MillMiscInfo(
| ${enclosingClasspath.map(p => literalize(p.toString))},
| ${literalize(scriptFolderPath.toString)},
| ${literalize(output.toString)},
| ${literalize(millTopLevelProjectRoot.toString)},
| _root_.scala.Seq(${segments.map(pprint.Util.literalize(_)).mkString(", ")})
|)
Expand Down
8 changes: 6 additions & 2 deletions runner/src/mill/runner/FileImportGraph.scala
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,11 @@ object FileImportGraph {
* starting from `build.mill`, collecting the information necessary to
* instantiate the [[MillRootModule]]
*/
def parseBuildFiles(topLevelProjectRoot: os.Path, projectRoot: os.Path): FileImportGraph = {
def parseBuildFiles(
topLevelProjectRoot: os.Path,
projectRoot: os.Path,
output: os.Path
): FileImportGraph = {
val seenScripts = mutable.Map.empty[os.Path, String]
val seenIvy = mutable.Set.empty[String]
val seenRepo = mutable.ListBuffer.empty[(String, os.Path)]
Expand Down Expand Up @@ -193,7 +197,7 @@ object FileImportGraph {
projectRoot,
followLinks = true,
skip = p =>
p == projectRoot / out ||
p == output ||
p == projectRoot / millBuild ||
(os.isDir(p) && !os.exists(p / nestedBuildFileName))
)
Expand Down
19 changes: 11 additions & 8 deletions runner/src/mill/runner/MillBuildBootstrap.scala
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import mill.eval.Evaluator
import mill.main.RunScript
import mill.resolve.SelectMode
import mill.define.{BaseModule, Discover, Segments}
import mill.main.client.OutFiles._
import mill.main.client.OutFiles.{millBuild, millRunnerState}

import java.net.URLClassLoader

Expand All @@ -30,6 +30,7 @@ import java.net.URLClassLoader
@internal
class MillBuildBootstrap(
projectRoot: os.Path,
output: os.Path,
home: os.Path,
keepGoing: Boolean,
imports: Seq[String],
Expand All @@ -46,15 +47,15 @@ class MillBuildBootstrap(
) {
import MillBuildBootstrap._

val millBootClasspath: Seq[os.Path] = prepareMillBootClasspath(projectRoot / out)
val millBootClasspath: Seq[os.Path] = prepareMillBootClasspath(output)
val millBootClasspathPathRefs: Seq[PathRef] = millBootClasspath.map(PathRef(_, quick = true))

def evaluate(): Watching.Result[RunnerState] = CliImports.withValue(imports) {
val runnerState = evaluateRec(0)

for ((frame, depth) <- runnerState.frames.zipWithIndex) {
os.write.over(
recOut(projectRoot, depth) / millRunnerState,
recOut(output, depth) / millRunnerState,
upickle.default.write(frame.loggedData, indent = 4),
createFolders = true
)
Expand Down Expand Up @@ -102,7 +103,8 @@ class MillBuildBootstrap(
} else {
val parsedScriptFiles = FileImportGraph.parseBuildFiles(
projectRoot,
recRoot(projectRoot, depth) / os.up
recRoot(projectRoot, depth) / os.up,
output
)

if (parsedScriptFiles.millImport) evaluateRec(depth + 1)
Expand All @@ -111,6 +113,7 @@ class MillBuildBootstrap(
new MillBuildRootModule.BootstrapModule(
projectRoot,
recRoot(projectRoot, depth),
output,
millBootClasspath
)(
mill.main.RootModule.Info(
Expand Down Expand Up @@ -340,8 +343,8 @@ class MillBuildBootstrap(
mill.eval.EvaluatorImpl(
home,
projectRoot,
recOut(projectRoot, depth),
recOut(projectRoot, depth),
recOut(output, depth),
recOut(output, depth),
rootModule,
PrefixLogger(logger, "", tickerContext = bootLogPrefix),
classLoaderSigHash = millClassloaderSigHash,
Expand Down Expand Up @@ -422,8 +425,8 @@ object MillBuildBootstrap {
projectRoot / Seq.fill(depth)(millBuild)
}

def recOut(projectRoot: os.Path, depth: Int): os.Path = {
projectRoot / out / Seq.fill(depth)(millBuild)
def recOut(output: os.Path, depth: Int): os.Path = {
output / Seq.fill(depth)(millBuild)
}

}
11 changes: 9 additions & 2 deletions runner/src/mill/runner/MillBuildRootModule.scala
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,8 @@ abstract class MillBuildRootModule()(implicit
parsed.seenScripts,
T.dest,
millBuildRootModuleInfo.enclosingClasspath,
millBuildRootModuleInfo.topLevelProjectRoot
millBuildRootModuleInfo.topLevelProjectRoot,
millBuildRootModuleInfo.output
)
Result.Success(Seq(PathRef(T.dest)))
}
Expand Down Expand Up @@ -265,12 +266,14 @@ object MillBuildRootModule {
class BootstrapModule(
topLevelProjectRoot0: os.Path,
projectRoot: os.Path,
output: os.Path,
enclosingClasspath: Seq[os.Path]
)(implicit baseModuleInfo: RootModule.Info) extends MillBuildRootModule()(
implicitly,
MillBuildRootModule.Info(
enclosingClasspath,
projectRoot,
output,
topLevelProjectRoot0
)
) {
Expand All @@ -281,25 +284,29 @@ object MillBuildRootModule {
case class Info(
enclosingClasspath: Seq[os.Path],
projectRoot: os.Path,
output: os.Path,
topLevelProjectRoot: os.Path
)

def parseBuildFiles(millBuildRootModuleInfo: MillBuildRootModule.Info): FileImportGraph = {
FileImportGraph.parseBuildFiles(
millBuildRootModuleInfo.topLevelProjectRoot,
millBuildRootModuleInfo.projectRoot / os.up
millBuildRootModuleInfo.projectRoot / os.up,
millBuildRootModuleInfo.output
)
}

class MillMiscInfo(
enclosingClasspath: Seq[String],
projectRoot: String,
output: String,
topLevelProjectRoot: String,
segments: Seq[String]
) {
implicit lazy val millBuildRootModuleInfo: MillBuildRootModule.Info = MillBuildRootModule.Info(
enclosingClasspath.map(os.Path(_)),
os.Path(projectRoot),
os.Path(output),
os.Path(topLevelProjectRoot)
)
implicit lazy val millBaseModuleInfo: RootModule.Info = RootModule.Info(
Expand Down
3 changes: 2 additions & 1 deletion runner/src/mill/runner/MillMain.scala
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import mill.java9rtexport.Export
import mill.api.{MillException, SystemStreams, WorkspaceRoot, internal}
import mill.bsp.{BspContext, BspServerResult}
import mill.main.BuildInfo
import mill.main.client.ServerFiles
import mill.main.client.{OutFiles, ServerFiles}
import mill.util.{PromptLogger, PrintLogger}

import java.lang.reflect.InvocationTargetException
Expand Down Expand Up @@ -234,6 +234,7 @@ object MillMain {

new MillBuildBootstrap(
projectRoot = WorkspaceRoot.workspaceRoot,
output = os.Path(OutFiles.out, WorkspaceRoot.workspaceRoot),
home = config.home,
keepGoing = config.keepGoing.value,
imports = config.imports,
Expand Down
2 changes: 1 addition & 1 deletion testkit/src/mill/testkit/IntegrationTester.scala
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ object IntegrationTester {
)
}

private val millTestSuiteEnv = Map("MILL_TEST_SUITE" -> this.getClass().toString())
def millTestSuiteEnv: Map[String, String] = Map("MILL_TEST_SUITE" -> this.getClass().toString())

/**
* Helpers to read the `.json` metadata files belonging to a particular task
Expand Down
Loading

0 comments on commit 4c3d2cb

Please sign in to comment.