Distribute mamba with workbench

.github/workflows/build-and-test.yml

@@ -364,6 +364,23 @@ jobs:
           yarn config set network-timeout 600000 -g
           yarn install
 
+      - name: Download mamba installer for distribution (MacOS)
+        if: matrix.os == 'macos-13'
+        run: curl -fsSLo Miniforge-pypy3-MacOSX-x86_64.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge-pypy3-MacOSX-x86_64.sh"
+
+      - name: Install mamba for MacOS
+        if: matrix.os == 'macos-13'
+        run: bash Miniforge-pypy3-MacOSX-x86_64.sh -b -p dist/miniforge3
+
+      - name: Download mamba installer for distribution (Windows)
+        if: matrix.os == 'windows-latest'
+        run: curl -fsSLo Miniforge-pypy3-Windows-x86_64.exe "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge-pypy3-Windows-x86_64.exe"
+
+      - name: Install mamba for Windows
+        if: matrix.os == 'windows-latest'
+        shell: cmd
+        run: Miniforge-pypy3-Windows-x86_64.exe /InstallationType=JustMe /RegisterPython=0 /S /D=%cd%\dist\miniforge3
+
       - name: Authenticate GCP
         if: github.event_name != 'pull_request'
         uses: google-github-actions/auth@v0

requirements-dev.txt

@@ -20,7 +20,7 @@ pypiwin32; sys_platform == 'win32'  # pip-only
 
 # 60.7.0 exception because of https://github.com/pyinstaller/pyinstaller/issues/6564
 setuptools>=8.0,!=60.7.0
-PyInstaller>=4.10  # pip-only
+PyInstaller>=6.9.0
 setuptools_scm>=6.4.0
 requests
 coverage

workbench/electron-builder-config.js

@@ -35,6 +35,10 @@ const config = {
       from: '../dist/invest',
       to: 'invest',
     },
+    {
+      from: '../dist/Miniforge3',
+      to: 'Miniforge3',
+    },
     {
       from: '../dist/userguide',
       to: 'documentation',

workbench/src/main/createPythonFlaskProcess.js

@@ -123,17 +123,14 @@ export async function createPluginServerProcess(modelName, _port = undefined) {
   }
 
   logger.debug('creating invest plugin server process');
-  const micromambaPath = 'micromamba' //settingsStore.get('micromamba_path');
+  const mamba = settingsStore.get('mamba');
   const modelEnvPath = settingsStore.get(`plugins.${modelName}.env`);
   const args = [
     'run', '--prefix', `"${modelEnvPath}"`,
     'invest', '--debug', 'serve', '--port', port];
-  logger.debug('spawning command:', micromambaPath, args);
-  const pythonServerProcess = spawn(
-    '"' + micromambaPath + '"',
-    args,
-    { shell: true } // necessary in dev mode & relying on a conda env
-  );
+  logger.debug('spawning command:', mamba, args);
+  // shell mode is necessary in dev mode & relying on a conda env
+  const pythonServerProcess = spawn(mamba, args, { shell: true });
   settingsStore.set(`plugins.${modelName}.port`, port);
   settingsStore.set(`plugins.${modelName}.pid`, pythonServerProcess.pid);
 

workbench/src/main/findInvestBinaries.js → workbench/src/main/findBinaries.js

@@ -1,10 +1,12 @@
-import path from 'path';
-import { spawnSync } from 'child_process';
+import upath from 'upath';
+import fs from 'fs';
+import { execSync, spawnSync } from 'child_process';
 
 import { ipcMain } from 'electron';
 
 import { ipcMainChannels } from './ipcMainChannels';
 import { getLogger } from './logger';
+import { checkFirstRun } from './setupCheckFirstRun';
 
 const logger = getLogger(__filename.split('/').slice(-1)[0]);
 
@@ -14,7 +16,7 @@ const logger = getLogger(__filename.split('/').slice(-1)[0]);
  * @param {boolean} isDevMode - a boolean designating dev mode or not.
  * @returns {string} invest binary path string.
  */
-export default function findInvestBinaries(isDevMode) {
+export function findInvestBinaries(isDevMode) {
   // Binding to the invest server binary:
   let investExe;
   const ext = (process.platform === 'win32') ? '.exe' : '';
@@ -23,7 +25,7 @@ export default function findInvestBinaries(isDevMode) {
   if (isDevMode) {
     investExe = filename; // assume an active python env w/ exe on path
   } else {
-    investExe = path.join(process.resourcesPath, 'invest', filename);
+    investExe = upath.join(process.resourcesPath, 'invest', filename);
     // It's likely the exe path includes spaces because it's composed of
     // app's Product Name, a user-facing name given to electron-builder.
     // Quoting depends on the shell, ('/bin/sh' or 'cmd.exe') and type of
@@ -51,3 +53,33 @@ export default function findInvestBinaries(isDevMode) {
   );
   return investExe;
 }
+
+
+/**
+ * Return the available mamba executable.
+ *
+ * @param {boolean} isDevMode - a boolean designating dev mode or not.
+ * @returns {string} mamba executable.
+ */
+export function findMambaExecutable(isDevMode) {
+  let mambaExe;
+  if (isDevMode) {
+    mambaExe = 'mamba'; // assume that mamba is available
+  } else {
+    if (process.platform === 'win32') {
+      mambaExe = `"${upath.join(process.resourcesPath, 'miniforge3', 'condabin', 'mamba.bat')}"`;
+    } else {
+      // Quote the path in case of spaces
+      mambaExe = `"${upath.join(process.resourcesPath, 'miniforge3', 'condabin', 'mamba')}"`;
+    }
+  }
+  // Check that the executable is working
+  const { stderr, error } = spawnSync(mambaExe, ['--help'], { shell: true });
+  if (error) {
+    logger.error(stderr.toString());
+    logger.error('mamba executable is not where we expected it.');
+    throw error;
+  }
+  logger.info(`using mamba executable '${mambaExe}'`);
+  return mambaExe;
+}

workbench/src/main/main.js

@@ -13,7 +13,7 @@ import {
   createCoreServerProcess,
   shutdownPythonProcess
 } from './createPythonFlaskProcess';
-import findInvestBinaries from './findInvestBinaries';
+import { findInvestBinaries, findMambaExecutable } from './findBinaries';
 import setupDownloadHandlers from './setupDownloadHandlers';
 import setupDialogs from './setupDialogs';
 import setupContextMenu from './setupContextMenu';
@@ -78,8 +78,8 @@ export const createWindow = async () => {
   });
   splashScreen.loadURL(path.join(BASE_URL, 'splash.html'));
 
-  const investExe = findInvestBinaries(ELECTRON_DEV_MODE);
-  settingsStore.set('investExe', investExe);
+  settingsStore.set('investExe', findInvestBinaries(ELECTRON_DEV_MODE));
+  settingsStore.set('mamba', findMambaExecutable(ELECTRON_DEV_MODE));
   // No plugin server processes should persist between workbench sessions
   // In case any were left behind, remove them
   const plugins = settingsStore.get('plugins');

workbench/src/main/setupAddPlugin.js

@@ -22,9 +22,9 @@ export default function setupAddPlugin() {
         const tmpPluginDir = fs.mkdtempSync(upath.join(tmpdir(), 'natcap-invest-'));
         execSync(
           `git clone --depth 1 --no-checkout ${pluginURL} "${tmpPluginDir}"`,
-          { stdio: 'inherit' }
+          { stdio: 'inherit', windowsHide: true }
         );
-        execSync('git checkout HEAD pyproject.toml', { cwd: tmpPluginDir, stdio: 'inherit' });
+        execSync('git checkout HEAD pyproject.toml', { cwd: tmpPluginDir, stdio: 'inherit', windowsHide: true });
 
         // Read in the plugin's pyproject.toml, then delete it
         const pyprojectTOML = toml.parse(fs.readFileSync(
@@ -39,13 +39,20 @@ export default function setupAddPlugin() {
 
         // Create a conda env containing the plugin and its dependencies
         const envName = `invest_plugin_${pluginID}`;
-        fs.writeFileSync('tmp_env.txt', 'python<3.12\ngdal<3.6');
-        execSync(`micromamba create --yes --name ${envName} -f tmp_env.txt -c conda-forge`);
-        execSync(`micromamba run --name ${envName} pip install "git+${pluginURL}"`, { stdio: 'inherit' });
+        const mamba = settingsStore.get('mamba');
+        execSync(`${mamba} create --yes --name ${envName} -c conda-forge "python<3.12" "gdal<3.6"`,
+          { stdio: 'inherit', windowsHide: true });
+        logger.info('created mamba env for plugin');
+        execSync(`${mamba} run --name ${envName} pip install "git+${pluginURL}"`,
+          { stdio: 'inherit', windowsHide: true });
+        logger.info('installed plugin into its env');
 
         // Write plugin metadata to the workbench's config.json
-        const envInfo = execSync(`micromamba info --name ${envName}`).toString();
-        const envPath = envInfo.split('env location : ')[1].split('\n')[0];
+        const envInfo = execSync(`${mamba} info --envs`, { windowsHide: true }).toString();
+        logger.info(`env info:\n${envInfo}`);
+        const envPath = envInfo.match(`${envName}\\s+(.+)$`)[1];
+        logger.info(`env path:\n${envPath}`);
+        logger.info('writing plugin info to settings store');
         settingsStore.set(
           `plugins.${pluginID}`,
           {

workbench/src/main/setupInvestHandlers.js

@@ -90,7 +90,7 @@ export function setupInvestRunHandlers() {
     let cmdArgs;
     const plugins = settingsStore.get('plugins');
     if (plugins && Object.keys(plugins).includes(modelRunName)) {
-      cmd = 'micromamba'//settingsStore.get('micromamba_path');
+      cmd = settingsStore.get('mamba');
       cmdArgs = [
         'run',
         `--prefix ${settingsStore.get(`plugins.${modelRunName}.env`)}`,

workbench/tests/invest/flaskapp.test.js

@@ -18,7 +18,7 @@ import {
   createCoreServerProcess,
   shutdownPythonProcess
 } from '../../src/main/createPythonFlaskProcess';
-import findInvestBinaries from '../../src/main/findInvestBinaries';
+import { findInvestBinaries } from '../../src/main/findBinaries';
 import { settingsStore } from '../../src/main/settingsStore';
 import { checkFirstRun, APP_HAS_RUN_TOKEN } from '../../src/main/setupCheckFirstRun';
 import { ipcMainChannels } from '../../src/main/ipcMainChannels';

workbench/tests/main/main.test.js

@@ -9,7 +9,7 @@
 
 import fs from 'fs';
 import os from 'os';
-import path from 'path';
+import upath from 'upath';
 import { spawnSync } from 'child_process';
 
 import { app, ipcMain } from 'electron';
@@ -30,7 +30,7 @@ import {
   createCoreServerProcess,
   getFlaskIsReady
 } from '../../src/main/createPythonFlaskProcess';
-import findInvestBinaries from '../../src/main/findInvestBinaries';
+import { findInvestBinaries } from '../../src/main/findBinaries';
 import extractZipInplace from '../../src/main/extractZipInplace';
 import { ipcMainChannels } from '../../src/main/ipcMainChannels';
 import investUsageLogger from '../../src/main/investUsageLogger';
@@ -55,7 +55,7 @@ beforeEach(() => {
 });
 
 describe('checkFirstRun', () => {
-  const tokenPath = path.join(app.getPath(), APP_HAS_RUN_TOKEN);
+  const tokenPath = upath.join(app.getPath(), APP_HAS_RUN_TOKEN);
   beforeEach(() => {
     try {
       fs.unlinkSync(tokenPath);
@@ -94,7 +94,7 @@ describe('findInvestBinaries', () => {
     const isDevMode = false;
     const exePath = findInvestBinaries(isDevMode);
     expect(exePath).toBe(
-      `"${path.join(process.resourcesPath, 'invest', filename)}"`
+      `"${upath.join(process.resourcesPath, 'invest', filename)}"`
     );
   });
 
@@ -114,27 +114,27 @@ describe('findInvestBinaries', () => {
 // probably means re-writing this test from scratch.
 const maybe = process.platform !== 'win32' ? describe : describe.skip;
 maybe('extractZipInplace', () => {
-  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'data-'));
-  const zipPath = path.join(root, 'output.zip');
+  const root = fs.mkdtempSync(upath.join(os.tmpdir(), 'data-'));
+  const zipPath = upath.join(root, 'output.zip');
   let level1Dir;
   let level2Dir;
   let file1Path;
   let file2Path;
   let doneZipping = false;
 
   beforeEach((done) => {
-    level1Dir = fs.mkdtempSync(path.join(root, 'level1'));
-    level2Dir = fs.mkdtempSync(path.join(level1Dir, 'level2'));
-    file1Path = path.join(level1Dir, 'file1');
-    file2Path = path.join(level2Dir, 'file2');
+    level1Dir = fs.mkdtempSync(upath.join(root, 'level1'));
+    level2Dir = fs.mkdtempSync(upath.join(level1Dir, 'level2'));
+    file1Path = upath.join(level1Dir, 'file1');
+    file2Path = upath.join(level2Dir, 'file2');
     fs.closeSync(fs.openSync(file1Path, 'w'));
     fs.closeSync(fs.openSync(file2Path, 'w'));
 
     const zipfile = new yazl.ZipFile();
     // adding the deeper file first, so extract function needs to
     // deal with extracting to non-existent directories.
-    zipfile.addFile(file2Path, path.relative(root, file2Path));
-    zipfile.addFile(file1Path, path.relative(root, file1Path));
+    zipfile.addFile(file2Path, upath.relative(root, file2Path));
+    zipfile.addFile(file1Path, upath.relative(root, file1Path));
     zipfile.outputStream.pipe(
       fs.createWriteStream(zipPath)
     ).on('close', () => {