Multithreading with compile

libgdx · Jul 21, 2024 · bc984de · bc984de
1 parent 0743ee7
commit bc984de
Show file tree

Hide file tree

Showing 8 changed files with 179 additions and 67 deletions.
diff --git a/gdx-jnigen-gradle/src/main/java/com/badlogic/gdx/jnigen/gradle/JnigenBuildTask.java b/gdx-jnigen-gradle/src/main/java/com/badlogic/gdx/jnigen/gradle/JnigenBuildTask.java
@@ -52,6 +52,8 @@ public void run () {
 
         BuildConfig buildConfig = new BuildConfig(ext.sharedLibName, ext.subProjectDir + ext.temporaryDir, ext.subProjectDir + ext.libsDir, ext.subProjectDir + ext.jniDir, robovmBuildConfig, new FileDescriptor(ext.subProjectDir));
 
+        buildConfig.multiThreadedCompile = ext.multiThreadedCompile;
+
         //Build the build target!
         //If we have no build target, its OS level build target build
 

diff --git a/gdx-jnigen-gradle/src/main/java/com/badlogic/gdx/jnigen/gradle/JnigenExtension.java b/gdx-jnigen-gradle/src/main/java/com/badlogic/gdx/jnigen/gradle/JnigenExtension.java
@@ -69,6 +69,8 @@ public class JnigenExtension {
      */
     boolean release = true;
 
+    boolean multiThreadedCompile = true;
+
     NativeCodeGeneratorConfig nativeCodeGeneratorConfig;
     public List<BuildTarget> targets = new ArrayList<>();
     Action<BuildTarget> all = null;

diff --git a/gdx-jnigen/src/main/java/com/badlogic/gdx/jnigen/BuildConfig.java b/gdx-jnigen/src/main/java/com/badlogic/gdx/jnigen/BuildConfig.java
@@ -62,6 +62,8 @@ public class BuildConfig {
 
     public RobovmBuildConfig robovmBuildConfig;
 
+    public boolean multiThreadedCompile = true;
+
     /**
      * Creates a new BuildConfig. The build directory, the libs directory and the jni directory are assumed to be "target", "libs"
      * and "jni". All paths are relative to the application's working directory.

diff --git a/gdx-jnigen/src/main/java/com/badlogic/gdx/jnigen/build/toolchains/AndroidToolchain.java b/gdx-jnigen/src/main/java/com/badlogic/gdx/jnigen/build/toolchains/AndroidToolchain.java
@@ -31,7 +31,7 @@ public void checkForTools () {
     }
 
     @Override
-    public void compile () {
+    public void compileTasks () {
         if (target.os != Os.Android) throw new IllegalArgumentException("target os must be Android");
 
         // create all the directories for outputing object files, shared libs and natives jar as well as build scripts.
@@ -102,6 +102,9 @@ public void compile () {
         config.buildDir.child("android32").child("Android.mk").writeString(template, false);
 
         ArrayList<String> args = new ArrayList<>();
+        if (config.multiThreadedCompile) {
+            args.add("-j" + Runtime.getRuntime().availableProcessors());
+        }
         args.add("NDK_PROJECT_PATH=" + buildDirectory.getAbsolutePath());
         args.add("NDK_APPLICATION_MK=" + new File(buildDirectory, "Application.mk").getAbsolutePath());
         args.add("APP_BUILD_SCRIPT=" + new File(buildDirectory, "Android.mk").getAbsolutePath());
@@ -111,4 +114,19 @@ public void compile () {
 
     }
 
+    @Override
+    public void collectCompilationTasks () {
+       //Dont have independent compilation tasks, ndk handles it
+    }
+
+    @Override
+    public void link () {
+        //NDK build handles
+    }
+
+    @Override
+    public void strip () {
+        //NDK build handles
+    }
+
 }
diff --git a/gdx-jnigen/src/main/java/com/badlogic/gdx/jnigen/build/toolchains/BaseToolchain.java b/gdx-jnigen/src/main/java/com/badlogic/gdx/jnigen/build/toolchains/BaseToolchain.java
@@ -17,6 +17,10 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 
 public abstract class BaseToolchain {
 
@@ -39,6 +43,12 @@ public abstract class BaseToolchain {
     protected File libsDirectory;
     protected File targetLibFile;
 
+    protected List<Callable<Void>> compilationTasks;
+
+    BaseToolchain () {
+        compilationTasks = new ArrayList<>();
+    }
+
     public void configure (BuildTarget target, BuildConfig config) {
         this.target = target;
         this.config = config;
@@ -59,10 +69,57 @@ public void build () {
         cleanBuildDirectory();
         createBuildDirectory();
 
-        compile();
+        collectCompilationTasks();
+        compileTasks();
+
+        link();
+        strip();
+    }
+
+    protected void compileTasks () {
+        if (config.multiThreadedCompile) {
+            logger.info("Compiling in multithreaded mode");
+
+            int threads = Runtime.getRuntime().availableProcessors();
+            ExecutorService executorService = Executors.newFixedThreadPool(threads);
+
+            ArrayList<Future<Void>> futures = new ArrayList<>();
+
+            for (Callable<Void> compilationTask : compilationTasks) {
+                Future<Void> submit = executorService.submit(compilationTask);
+                futures.add(submit);
+            }
+
+            //Wait for all futures to complete
+            for (Future<Void> future : futures) {
+                try {
+                    future.get();
+                } catch (Exception e) {
+                    logger.error("Failure to compile task", e);
+                    throw new RuntimeException("Failure to compile task", e);
+                }
+            }
+            executorService.shutdown();
+
+
+        } else {
+            logger.info("Compiling in single threaded mode");
+
+            for (Callable<Void> task : compilationTasks) {
+                try {
+                    task.call();
+                } catch (Exception e) {
+                    logger.error("Failure to compile task", e);
+                    throw new RuntimeException("Failure to compile task", e);
+                }
+            }
+        }
     }
 
-    public abstract void compile ();
+    public abstract void collectCompilationTasks ();
+
+    public abstract void link ();
+    public abstract void strip ();
 
     private void collectCFiles () {
         File jniDir = config.jniDir.file();

diff --git a/gdx-jnigen/src/main/java/com/badlogic/gdx/jnigen/build/toolchains/GNUToolchain.java b/gdx-jnigen/src/main/java/com/badlogic/gdx/jnigen/build/toolchains/GNUToolchain.java
@@ -9,6 +9,7 @@
 import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.Callable;
 
 /**
  * Toolchain that compiles c/c++ using gcc/g++/clang toolchains
@@ -42,16 +43,13 @@ public void checkForTools () {
 
 
     @Override
-    public void compile () {
-        compileC();
-        compileCPP();
-
-        link();
-        strip();
+    public void collectCompilationTasks () {
+        collectCCompileTasks();
+        collectCPPCompileTasks();
     }
 
 
-    private void compileC () {
+    private void collectCCompileTasks () {
         if (buildCFiles.isEmpty()) {
             logger.info("No c files, skipping c compilation");
             return;
@@ -67,23 +65,27 @@ private void compileC () {
             args.add("-I" + convertToAbsoluteRelativeTo(config.projectDir, headerDir));
         }
 
-        ArrayList<String> perFileArgs = new ArrayList<String>();
-        perFileArgs.addAll(args);
 
         for (File buildCFile : buildCFiles) {
-            perFileArgs.clear();
+            ArrayList<String> perFileArgs = new ArrayList<String>();
             perFileArgs.addAll(args);
 
             perFileArgs.add(buildCFile.getAbsolutePath());
             perFileArgs.add("-o");
             perFileArgs.add(new File(buildDirectory, toObjectFile(buildCFile.getName())).getAbsolutePath());
 
-            logger.info("Compiling C File {}", buildCFile.getAbsolutePath());
-            ToolchainExecutor.execute(cCompilerExecutable, config.jniDir.file(), perFileArgs, createToolChainCallback("C Compile"));
+            compilationTasks.add(new Callable<Void>() {
+                @Override
+                public Void call () throws Exception {
+                    logger.info("Compiling C File {}", buildCFile.getAbsolutePath());
+                    ToolchainExecutor.execute(cCompilerExecutable, config.jniDir.file(), perFileArgs, createToolChainCallback("C Compile - " + buildCFile.getName()));
+                    return null;
+                }
+            });
         }
     }
 
-    private void compileCPP () {
+    private void collectCPPCompileTasks () {
         if (buildCppFiles.isEmpty()) {
             logger.info("No c files, skipping c compilation");
             return;
@@ -99,19 +101,23 @@ private void compileCPP () {
             args.add("-I" + convertToAbsoluteRelativeTo(config.projectDir, headerDir));
         }
 
-        ArrayList<String> perFileArgs = new ArrayList<String>();
-        perFileArgs.addAll(args);
-
         for (File buildCppFile : buildCppFiles) {
-            perFileArgs.clear();
+            ArrayList<String> perFileArgs = new ArrayList<String>();
             perFileArgs.addAll(args);
 
             perFileArgs.add(buildCppFile.getAbsolutePath());
             perFileArgs.add("-o");
             perFileArgs.add(new File(buildDirectory, toObjectFile(buildCppFile.getName())).getAbsolutePath());
 
-            logger.info("Compiling CPP File {}", buildCppFile.getAbsolutePath());
-            ToolchainExecutor.execute(cppCompilerExecutable, config.jniDir.file(), perFileArgs, createToolChainCallback("CPP Compile"));
+            compilationTasks.add(new Callable<Void>() {
+                @Override
+                public Void call () throws Exception {
+                    logger.info("Compiling CPP File {}", buildCppFile.getAbsolutePath());
+                    ToolchainExecutor.execute(cppCompilerExecutable, config.jniDir.file(), perFileArgs, createToolChainCallback("CPP Compile - " + buildCppFile.getName()));
+                    return null;
+                }
+            });
+
         }
 
     }
@@ -121,7 +127,8 @@ private String toObjectFile (String name) {
         return split[0] + ".o";
     }
 
-    private void link () {
+    @Override
+    public void link () {
 
         libsDirectory.mkdirs();
         if (!libsDirectory.exists()) {
@@ -153,7 +160,8 @@ private void link () {
         ToolchainExecutor.execute(linkerExecutable, config.buildDir.file(), args, createToolChainCallback("Link"));
     }
 
-    private void strip () {
+    @Override
+    public void strip () {
         if (!target.release) {
             logger.info("Skipping strip due to release property {}", target.release);
             return;

diff --git a/gdx-jnigen/src/main/java/com/badlogic/gdx/jnigen/build/toolchains/IOSToolchain.java b/gdx-jnigen/src/main/java/com/badlogic/gdx/jnigen/build/toolchains/IOSToolchain.java
@@ -10,6 +10,7 @@
 import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.Callable;
 
 /**
  * Toolchain that compiles c/c++ using gcc/g++/clang toolchains
@@ -42,11 +43,9 @@ public void checkForTools () {
 
 
     @Override
-    public void compile () {
-        compileC();
-        compileCPP();
-
-        link();
+    public void collectCompilationTasks () {
+        collectCCompileTasks();
+        collectCPPCompileTasks();
     }
 
     private String getClangArch () {
@@ -60,7 +59,7 @@ private String getClangArch () {
     }
 
 
-    private void compileC () {
+    private void collectCCompileTasks () {
         if (buildCFiles.isEmpty()) {
             logger.info("No c files, skipping c compilation");
             return;
@@ -92,23 +91,27 @@ private void compileC () {
 
         args.add("-g");
 
-        ArrayList<String> perFileArgs = new ArrayList<String>();
-        perFileArgs.addAll(args);
-
         for (File buildCFile : buildCFiles) {
-            perFileArgs.clear();
+            ArrayList<String> perFileArgs = new ArrayList<String>();
             perFileArgs.addAll(args);
 
             perFileArgs.add(buildCFile.getAbsolutePath());
             perFileArgs.add("-o");
             perFileArgs.add(new File(buildDirectory, toObjectFile(buildCFile.getName())).getAbsolutePath());
 
-            logger.info("Compiling C File {}", buildCFile.getAbsolutePath());
-            ToolchainExecutor.execute(cCompilerExecutable, config.jniDir.file(), perFileArgs, createToolChainCallback("C Compile"));
+
+            compilationTasks.add(new Callable<Void>() {
+                @Override
+                public Void call () throws Exception {
+                    logger.info("Compiling C File {}", buildCFile.getAbsolutePath());
+                    ToolchainExecutor.execute(cCompilerExecutable, config.jniDir.file(), perFileArgs, createToolChainCallback("C Compile - " + buildCFile.getName()));
+                    return null;
+                }
+            });
         }
     }
 
-    private void compileCPP () {
+    private void collectCPPCompileTasks () {
         if (buildCppFiles.isEmpty()) {
             logger.info("No c files, skipping c compilation");
             return;
@@ -141,19 +144,22 @@ private void compileCPP () {
 
         args.add("-g");
 
-        ArrayList<String> perFileArgs = new ArrayList<String>();
-        perFileArgs.addAll(args);
-
         for (File buildCppFile : buildCppFiles) {
-            perFileArgs.clear();
+            ArrayList<String> perFileArgs = new ArrayList<String>();
             perFileArgs.addAll(args);
 
             perFileArgs.add(buildCppFile.getAbsolutePath());
             perFileArgs.add("-o");
             perFileArgs.add(new File(buildDirectory, toObjectFile(buildCppFile.getName())).getAbsolutePath());
 
-            logger.info("Compiling CPP File {}", buildCppFile.getAbsolutePath());
-            ToolchainExecutor.execute(cppCompilerExecutable, config.jniDir.file(), perFileArgs, createToolChainCallback("CPP Compile"));
+            compilationTasks.add(new Callable<Void>() {
+                @Override
+                public Void call () throws Exception {
+                    logger.info("Compiling CPP File {}", buildCppFile.getAbsolutePath());
+                    ToolchainExecutor.execute(cppCompilerExecutable, config.jniDir.file(), perFileArgs, createToolChainCallback("CPP Compile - " + buildCppFile.getName()));
+                    return null;
+                }
+            });
         }
 
     }
@@ -163,7 +169,8 @@ private String toObjectFile (String name) {
         return split[0] + ".o";
     }
 
-    private void link () {
+    @Override
+    public void link () {
 
         libsDirectory.mkdirs();
         if (!libsDirectory.exists()) {
@@ -208,5 +215,10 @@ private void link () {
         ToolchainExecutor.execute(linkerExecutable, config.buildDir.file(), args, createToolChainCallback("Link"));
     }
 
+    @Override
+    public void strip () {
+
+    }
+
 
 }