Merge pull request #1071 from NvTimLiu/release-tmp

Merge branch 'branch-23.04' into main [skip ci]

.github/workflows/auto-merge.yml

@@ -18,12 +18,12 @@ name: auto-merge HEAD to BASE
 on:
   pull_request_target:
     branches:
-      - branch-23.02
+      - branch-23.04
     types: [closed]
 
 env:
-  HEAD: branch-23.02
-  BASE: branch-23.04
+  HEAD: branch-23.04
+  BASE: branch-23.06
 
 jobs:
   auto-merge:

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "thirdparty/cudf"]
 	path = thirdparty/cudf
 	url = https://github.com/rapidsai/cudf.git
-	branch = branch-23.02
+	branch = branch-23.04

CONTRIBUTING.md

@@ -144,7 +144,7 @@ $ ./build/build-in-docker install ...
 ```
 
 Now cd to ~/repos/NVIDIA/spark-rapids and build with one of the options from
-[spark-rapids instructions](https://github.com/NVIDIA/spark-rapids/blob/branch-23.02/CONTRIBUTING.md#building-from-source).
+[spark-rapids instructions](https://github.com/NVIDIA/spark-rapids/blob/branch-23.04/CONTRIBUTING.md#building-from-source).
 
 ```bash
 $ ./build/buildall

build/run-in-docker

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 #
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -74,4 +74,4 @@ $DOCKER_CMD run $DOCKER_GPU_OPTS $DOCKER_RUN_EXTRA_ARGS -u $(id -u):$(id -g) --r
   -e VERBOSE \
   $DOCKER_OPTS \
   $SPARK_IMAGE_NAME \
-  scl enable devtoolset-9 "$RUN_CMD"
+  scl enable devtoolset-11 "$RUN_CMD"

ci/Dockerfile

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,7 +22,7 @@
 ###
 ARG CUDA_VERSION=11.8.0
 FROM nvidia/cuda:$CUDA_VERSION-devel-centos7
-ARG DEVTOOLSET_VERSION=9
+ARG DEVTOOLSET_VERSION=11
 ### Install basic requirements
 RUN yum install -y centos-release-scl
 RUN yum install -y devtoolset-${DEVTOOLSET_VERSION} rh-python38 epel-release

ci/Jenkinsfile.premerge

@@ -1,6 +1,6 @@
 #!/usr/local/env groovy
 /*
- * Copyright (c) 2022, NVIDIA CORPORATION.
+ * Copyright (c) 2022-2023, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -211,7 +211,8 @@ pipeline {
                     container('gpu') {
                         timeout(time: 3, unit: 'HOURS') { // step only timeout for test run
                             common.resolveIncompatibleDriverIssue(this)
-                            sh 'scl enable devtoolset-9 "ci/premerge-build.sh"'
+                            sh 'scl enable devtoolset-11 "ci/premerge-build.sh"'
+                            sh 'bash ci/fuzz-test.sh'
                         }
                     }
                 }

ci/fuzz-test.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# NOTE:
+#     run fuzz test after build
+#     required jars: jni, jni-tests, slf4j-api
+
+set -ex
+
+WORKSPACE=${WORKSPACE:-$PWD}
+M2DIR=${M2DIR:-"$HOME/.m2"}
+
+SLF4J_VER=$(mvn help:evaluate -Dexpression=slf4j.version -q -DforceStdout)
+CLASSPATH=${CLASSPATH:-"$WORKSPACE/target/*:$M2DIR/repository/org/slf4j/slf4j-api/$SLF4J_VER/slf4j-api-$SLF4J_VER.jar"}
+
+java -cp "$CLASSPATH" \
+  com.nvidia.spark.rapids.jni.RmmSparkMonteCarlo \
+  --taskMaxMiB=2048 --gpuMiB=3072 --skewed --allocMode=ASYNC

ci/nightly-build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,11 +22,13 @@ nvidia-smi
 git submodule update --init --recursive
 
 MVN="mvn -Dmaven.wagon.http.retryHandler.count=3 -B"
+# cuda11 or cuda12
+CUDA_VER=${CUDA_VER:-cuda`nvcc --version | sed -n 's/^.*release \([0-9]\+\)\..*$/\1/p'`}
 PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
 USE_GDS=${USE_GDS:-ON}
 ${MVN} clean package ${MVN_MIRROR}  \
   -Psource-javadoc \
   -DCPP_PARALLEL_LEVEL=${PARALLEL_LEVEL} \
   -Dlibcudf.build.configure=true \
   -DUSE_GDS=${USE_GDS} -Dtest=*,!CuFileTest,!CudaFatalTest \
-  -DBUILD_TESTS=ON
+  -DBUILD_TESTS=ON -Dcuda.version=$CUDA_VER

ci/submodule-sync.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@
 # NOTE:
 #     this script is for jenkins only, and should not be used for local development
 #     run with ci/Dockerfile in jenkins:
-#         scl enable devtoolset-9 rh-python38 "ci/submodule-sync.sh"
+#         scl enable devtoolset-11 rh-python38 "ci/submodule-sync.sh"
 
 set -ex
 
@@ -69,7 +69,8 @@ set +e
 ${MVN} verify ${MVN_MIRROR} \
   -DCPP_PARALLEL_LEVEL=${PARALLEL_LEVEL} \
   -Dlibcudf.build.configure=true \
-  -DUSE_GDS=ON -Dtest=*,!CuFileTest,!CudaFatalTest
+  -DUSE_GDS=ON -Dtest=*,!CuFileTest,!CudaFatalTest \
+  -DBUILD_TESTS=ON
 verify_status=$?
 set -e
 

pom.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright (c) 2022, NVIDIA CORPORATION.
+  Copyright (c) 2022-2023, NVIDIA CORPORATION.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@
 
   <groupId>com.nvidia</groupId>
   <artifactId>spark-rapids-jni</artifactId>
-  <version>23.02.0</version>
+  <version>23.04.0</version>
   <packaging>jar</packaging>
   <name>RAPIDS Accelerator JNI for Apache Spark</name>
   <description>
@@ -118,6 +118,16 @@
       <artifactId>hadoop-common</artifactId>
       <version>${hadoop.version}</version>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-reload4j</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.parquet</groupId>
@@ -492,6 +502,7 @@
           <!-- you can turn this off, by passing -DtrimStackTrace=true when running tests -->
           <trimStackTrace>false</trimStackTrace>
           <redirectTestOutputToFile>true</redirectTestOutputToFile>
+          <runOrder>alphabetical</runOrder>
           <systemPropertyVariables>
             <ai.rapids.refcount.debug>${ai.rapids.refcount.debug}</ai.rapids.refcount.debug>
             <ai.rapids.cudf.nvtx.enabled>${ai.rapids.cudf.nvtx.enabled}</ai.rapids.cudf.nvtx.enabled>
@@ -591,6 +602,13 @@
         <directory>${project.build.directory}/extra-resources</directory>
         <filtering>true</filtering>
       </resource>
+      <resource>
+        <directory>${project.basedir}</directory>
+        <targetPath>META-INF</targetPath>
+        <includes>
+          <include>LICENSE</include>
+        </includes>
+      </resource>
     </resources>
   </build>
 </project>

src/main/cpp/CMakeLists.txt

@@ -32,7 +32,7 @@ rapids_cuda_init_architectures(SPARK_RAPIDS_JNI)
 
 project(
   SPARK_RAPIDS_JNI
-  VERSION 23.02.00
+  VERSION 23.04.00
   LANGUAGES C CXX CUDA
 )
 
@@ -145,6 +145,7 @@ add_library(
   src/MapUtilsJni.cpp
   src/NativeParquetJni.cpp
   src/RowConversionJni.cpp
+  src/SparkResourceAdaptorJni.cpp
   src/ZOrderJni.cpp
   src/cast_string.cu
   src/cast_string_to_float.cu

src/main/cpp/faultinj/CMakeLists.txt

@@ -1,5 +1,5 @@
 #=============================================================================
-# Copyright (c) 2022, NVIDIA CORPORATION.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,20 +22,15 @@ project(
 
 set(CMAKE_CXX_STANDARD 17)
 
-find_package(Boost REQUIRED)
-
 add_library(
   cufaultinj SHARED
   faultinj.cu
 )
 
-find_path(SPDLOG_INCLUDE "spdlog"
-    HINTS "$ENV{RMM_ROOT}/_deps/spdlog-src/include")
-
-include_directories(
-  "${SPDLOG_INCLUDE}"
+target_link_libraries(
+  cufaultinj PRIVATE spdlog::spdlog_header_only
 )
 
 target_link_libraries(
-  cufaultinj CUDA::cupti_static
+  cufaultinj PRIVATE CUDA::cupti_static
 )