[openmp] parallel learning using OpenMP

zouzias · Oct 21, 2019 · 0aff6c2 · 0aff6c2
1 parent d2bbfb2
commit 0aff6c2
Show file tree

Hide file tree

Showing 7 changed files with 86 additions and 19 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,7 +1,9 @@
-cmake_minimum_required(VERSION 3.5 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
 
 PROJECT(microgbt CXX)
 
+OPTION(USE_OPENMP "Enable OpenMP" ON)
+
 # project version
 set(VERSION_MAJOR 0)
 set(VERSION_MINOR 1)
@@ -17,6 +19,40 @@ SET(CMAKE_POSITION_INDEPENDENT_CODE ON)
 FIND_PACKAGE (Eigen3 3.3 REQUIRED NO_MODULE)
 
 
+# Based on https://iscinumpy.gitlab.io/post/omp-on-high-sierra/
+# and for Mojave 10.14: see https://github.com/microsoft/LightGBM/pull/1919
+if(USE_OPENMP)
+    find_package(OpenMP)
+
+    # If OpenMP wasn't found, try if we can find it in the default Homebrew location
+    if((NOT OPENMP_FOUND) AND (NOT OPENMP_CXX_FOUND) AND EXISTS "/usr/local/opt/libomp/lib/libomp.dylib")
+        set(OpenMP_C_FLAGS "-Xpreprocessor -fopenmp -I/usr/local/opt/libomp/include")
+        set(OpenMP_C_LIB_NAMES omp)
+        set(OpenMP_CXX_FLAGS "-Xpreprocessor -fopenmp -I/usr/local/opt/libomp/include")
+        set(OpenMP_CXX_LIB_NAMES omp)
+        set(OpenMP_omp_LIBRARY /usr/local/opt/libomp/lib/libomp.dylib)
+
+        find_package(OpenMP)
+        if (OPENMP_FOUND OR OPENMP_CXX_FOUND)
+            message(STATUS "Found libomp in homebrew default location.")
+        else()
+            message(FATAL_ERROR "Didn't find libomp. Tried homebrew default location but also didn't find it.")
+        endif()
+    endif()
+
+    if((NOT OPENMP_FOUND) AND (NOT OPENMP_CXX_FOUND))
+        message(FATAL_ERROR "Did not find OpenMP.")
+    endif()
+
+    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+else()
+    # Ignore unknown #pragma warning
+    if((CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+            OR (CMAKE_CXX_COMPILER_ID STREQUAL "GNU"))
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-pragmas")
+    endif()
+endif(USE_OPENMP)
+
 # place binaries and libraries according to GNU standards
 include(GNUInstallDirs)
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR})
@@ -72,3 +108,12 @@ pybind11_add_module(microgbtpy src/metrics/metric.h src/trees/tree.h src/GBT.h s
         src/utils.h
         src/python_api.cpp)
 #########################
+
+
+#########################
+# Link openMP
+#########################
+if(USE_OPENMP)
+    target_link_libraries(microgbtpy PRIVATE OpenMP::OpenMP_CXX)
+endif(USE_OPENMP)
+#########################
diff --git a/src/GBT.h b/src/GBT.h
@@ -2,6 +2,7 @@
 #include <vector>
 #include<iostream>
 #include <memory>
+#include <omp.h>
 #include "dataset.h"
 #include "trees/tree.h"
 #include "metrics/metric.h"
@@ -117,6 +118,9 @@ namespace microgbt {
          */
         void train(const Dataset &trainSet, const Dataset &validSet, int numBoostRound, int earlyStoppingRounds) {
 
+            // Allow nested threading in OpenMP
+            omp_set_max_active_levels(_maxDepth + 1);
+
             long bestIteration = 0;
             double learningRate = _shrinkageRate;
             double bestValidationLoss = std::numeric_limits<double>::max();

diff --git a/src/dataset.h b/src/dataset.h
@@ -69,6 +69,7 @@ namespace microgbt {
             // By default, all rows are included in the dataset
             std::iota(_rowIndices.begin(), _rowIndices.end(), 0);
 
+            #pragma omp parallel for schedule(static)
             for ( long j = 0; j < X.cols(); j++) {
                 _sortedMatrixIdx.col(j) = sortIndices(j);
             }

diff --git a/src/metrics/logloss.h b/src/metrics/logloss.h
@@ -51,6 +51,7 @@ namespace microgbt {
             unsigned long sz = predictions.size();
             Vector gradients(sz);
 
+            #pragma omp parallel for schedule(static)
             for (unsigned long i = 0 ; i < sz; i++){
                 gradients[i] = predictions[i] - labels[i];
             }
@@ -62,6 +63,7 @@ namespace microgbt {
             unsigned long sz = predictions.size();
             Vector hessians(sz);
 
+            #pragma omp parallel for schedule(static)
             for (unsigned long i = 0 ; i < sz; i++){
                 hessians[i] = predictions[i] * (1 - predictions[i]);
             }
@@ -73,6 +75,7 @@ namespace microgbt {
             size_t n = predictions.size();
             double loss = 0.0;
 
+            #pragma omp parallel for shared(labels, predictions) reduction(+: loss)
             for (size_t i = 0; i < n; i ++){
                 loss += labels[i] * log(predictions[i]) + (1 - labels[i]) * log(1 - predictions[i]);
             }

diff --git a/src/trees/numerical_splliter.h b/src/trees/numerical_splliter.h
@@ -91,6 +91,7 @@ namespace microgbt {
 
             // For each feature, compute split gain and keep the split index with maximum gain
             Vector gainPerOrderedSampleIndex(dataset.nRows());
+            #pragma omp parallel for schedule(static, 1024)
             for (size_t i = 0 ; i < dataset.nRows(); i++){
                 gainPerOrderedSampleIndex[i] = calc_split_gain(cum_sum_g, cum_sum_h, cum_sum_G[i], cum_sum_H[i]);
             }

diff --git a/src/trees/treenode.h b/src/trees/treenode.h
@@ -123,23 +123,34 @@ namespace microgbt {
             this->splitFeatureIndex = bestGain.getBestFeatureId();
             this->splitNumericValue = bestGain.splitValue();
 
-
-            Dataset leftDataset(trainSet, bestGain, SplitInfo::Side::Left);
-            Vector leftGradient = bestGain.split(gradient, SplitInfo::Side::Left);
-            Vector leftHessian = bestGain.split(hessian, SplitInfo::Side::Left);
-            Vector leftPreviousPreds = bestGain.split(previousPreds, SplitInfo::Side::Left);
-            this->leftSubTree = std::unique_ptr<TreeNode>(
-                    new TreeNode(_lambda, _minSplitGain, _minTreeSize, _maxDepth));
-            leftSubTree->build(leftDataset, leftPreviousPreds, leftGradient, leftHessian, shrinkage, depth + 1);
-
-            Dataset rightDataset(trainSet, bestGain, SplitInfo::Side::Right);
-            Vector rightGradient = bestGain.split(gradient, SplitInfo::Side::Right);
-            Vector rightHessian = bestGain.split(hessian, SplitInfo::Side::Right);
-            Vector rightPreviousPreds = bestGain.split(previousPreds, SplitInfo::Side::Right);
-
-            this->rightSubTree = std::unique_ptr<TreeNode>(
-                    new TreeNode(_lambda, _minSplitGain, _minTreeSize, _maxDepth));
-            rightSubTree->build(rightDataset, rightPreviousPreds, rightGradient, rightHessian, shrinkage, depth + 1);
+            #pragma omp parallel sections
+            {
+                // Recurse on the left subtree
+                #pragma omp section
+                {
+                    Dataset leftDataset(trainSet, bestGain, SplitInfo::Side::Left);
+                    Vector leftGradient = bestGain.split(gradient, SplitInfo::Side::Left);
+                    Vector leftHessian = bestGain.split(hessian, SplitInfo::Side::Left);
+                    Vector leftPreviousPreds = bestGain.split(previousPreds, SplitInfo::Side::Left);
+                    this->leftSubTree = std::unique_ptr<TreeNode>(
+                            new TreeNode(_lambda, _minSplitGain, _minTreeSize, _maxDepth));
+                    leftSubTree->build(leftDataset, leftPreviousPreds, leftGradient, leftHessian, shrinkage, depth + 1);
+                }
+
+
+                // Recurse on the right subtree
+                #pragma omp section
+                {
+                    Dataset rightDataset(trainSet, bestGain, SplitInfo::Side::Right);
+                    Vector rightGradient = bestGain.split(gradient, SplitInfo::Side::Right);
+                    Vector rightHessian = bestGain.split(hessian, SplitInfo::Side::Right);
+                    Vector rightPreviousPreds = bestGain.split(previousPreds, SplitInfo::Side::Right);
+
+                    this->rightSubTree = std::unique_ptr<TreeNode>(
+                            new TreeNode(_lambda, _minSplitGain, _minTreeSize, _maxDepth));
+                    rightSubTree->build(rightDataset, rightPreviousPreds, rightGradient, rightHessian, shrinkage, depth + 1);
+                }
+            }
         }
 
         /**

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -12,7 +12,9 @@ add_executable(
 target_link_libraries(
     unit_tests
     gtest_main
-    microgbt)
+    microgbt
+    OpenMP::OpenMP_CXX
+    )
 
 add_test(
   NAME