From 8abf4ac6e3c5bdac42868caacd20ab365e665d02 Mon Sep 17 00:00:00 2001 From: Paul Taylor Date: Wed, 25 Aug 2021 13:16:07 -0500 Subject: [PATCH 1/3] Remove -g from cython compile commands (#1783) Removes `-g` from the compile commands generated by distutils to compile Cython files. This will make our container images, conda packages, and python wheels smaller. --- python/setup.py | 123 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 83 insertions(+), 40 deletions(-) diff --git a/python/setup.py b/python/setup.py index 6009e56d7f1..a2dde239cca 100644 --- a/python/setup.py +++ b/python/setup.py @@ -16,18 +16,27 @@ import sysconfig import shutil -from setuptools import setup, find_packages, Command -from setuptools.extension import Extension -from setuputils import use_raft_package, get_environment_option +# Must import in this order: +# setuptools -> Cython.Distutils.build_ext -> setuptools.command.build_ext +# Otherwise, setuptools.command.build_ext ends up inheriting from +# Cython.Distutils.old_build_ext which we do not want +import setuptools try: - from Cython.Distutils.build_ext import new_build_ext as build_ext + from Cython.Distutils.build_ext import new_build_ext as _build_ext except ImportError: - from setuptools.command.build_ext import build_ext + from setuptools.command.build_ext import build_ext as _build_ext -import versioneer from distutils.sysconfig import get_python_lib +import setuptools.command.build_ext +from setuptools import find_packages, setup, Command +from setuptools.extension import Extension + +from setuputils import use_raft_package, get_environment_option + +import versioneer + INSTALL_REQUIRES = ['numba', 'cython'] CYTHON_FILES = ['cugraph/**/*.pyx'] @@ -81,6 +90,32 @@ if not libcugraph_path: libcugraph_path = conda_lib_dir +extensions = [ + Extension("*", + sources=CYTHON_FILES, + include_dirs=[ + conda_include_dir, + ucx_include_dir, + '../cpp/include', + "../thirdparty/cub", + raft_include_dir, + os.path.join(conda_include_dir, "libcudacxx"), + cuda_include_dir, + os.path.dirname(sysconfig.get_path("include")) + ], + library_dirs=[ + get_python_lib(), + conda_lib_dir, + libcugraph_path, + ucx_lib_dir, + cuda_lib_dir, + os.path.join(os.sys.prefix, "lib") + ], + libraries=['cudart', 'cusparse', 'cusolver', 'cugraph', 'nccl'], + language='c++', + extra_compile_args=['-std=c++17']) +] + class CleanCommand(Command): """Custom clean command to tidy up the project root.""" @@ -105,42 +140,50 @@ def run(self): os.system('find . -name "*.cpython*.so" -type f -delete') +class build_ext_no_debug(_build_ext): + + def build_extensions(self): + def remove_flags(compiler, *flags): + for flag in flags: + try: + compiler.compiler_so = list( + filter((flag).__ne__, compiler.compiler_so) + ) + except Exception: + pass + # Full optimization + self.compiler.compiler_so.append("-O3") + # No debug symbols, full optimization, no '-Wstrict-prototypes' warning + remove_flags( + self.compiler, "-g", "-G", "-O1", "-O2", "-Wstrict-prototypes" + ) + super().build_extensions() + + def finalize_options(self): + if self.distribution.ext_modules: + # Delay import this to allow for Cython-less installs + from Cython.Build.Dependencies import cythonize + + nthreads = getattr(self, "parallel", None) # -j option in Py3.5+ + nthreads = int(nthreads) if nthreads else None + self.distribution.ext_modules = cythonize( + self.distribution.ext_modules, + nthreads=nthreads, + force=self.force, + gdb_debug=False, + compiler_directives=dict( + profile=False, language_level=3, embedsignature=True + ), + ) + # Skip calling super() and jump straight to setuptools + setuptools.command.build_ext.build_ext.finalize_options(self) + + cmdclass = dict() cmdclass.update(versioneer.get_cmdclass()) -cmdclass["build_ext"] = build_ext +cmdclass["build_ext"] = build_ext_no_debug cmdclass["clean"] = CleanCommand -EXTENSIONS = [ - Extension("*", - sources=CYTHON_FILES, - include_dirs=[ - conda_include_dir, - ucx_include_dir, - '../cpp/include', - "../thirdparty/cub", - raft_include_dir, - os.path.join(conda_include_dir, "libcudacxx"), - cuda_include_dir, - os.path.dirname(sysconfig.get_path("include")) - ], - library_dirs=[ - get_python_lib(), - conda_lib_dir, - libcugraph_path, - ucx_lib_dir, - cuda_lib_dir, - os.path.join(os.sys.prefix, "lib") - ], - libraries=['cudart', 'cusparse', 'cusolver', 'cugraph', 'nccl'], - language='c++', - extra_compile_args=['-std=c++17']) -] - -for e in EXTENSIONS: - e.cython_directives = dict( - profile=False, language_level=3, embedsignature=True - ) - setup(name='cugraph', description="cuGraph - GPU Graph Analytics", version=versioneer.get_version(), @@ -154,8 +197,8 @@ def run(self): ], # Include the separately-compiled shared library author="NVIDIA Corporation", - setup_requires=['cython'], - ext_modules=EXTENSIONS, + setup_requires=['Cython>=0.29,<0.30'], + ext_modules=extensions, packages=find_packages(include=['cugraph', 'cugraph.*']), install_requires=INSTALL_REQUIRES, license="Apache", From b681f73665fc6576341974c8dd377b401a1bdd32 Mon Sep 17 00:00:00 2001 From: Chuck Hastings <45364586+ChuckHastings@users.noreply.github.com> Date: Thu, 16 Sep 2021 11:48:29 -0400 Subject: [PATCH 2/3] update TSP with Texas State Copyright (#1814) --- cpp/src/traversal/tsp.cu | 1 + cpp/src/traversal/tsp.hpp | 1 + cpp/src/traversal/tsp_solver.hpp | 1 + cpp/src/traversal/tsp_utils.hpp | 1 + .../LICENSES/LICENSE.texas_state_university | 24 +++++++++++++++++++ 5 files changed, 28 insertions(+) create mode 100644 thirdparty/LICENSES/LICENSE.texas_state_university diff --git a/cpp/src/traversal/tsp.cu b/cpp/src/traversal/tsp.cu index 332ccb21834..9be4f4f3767 100644 --- a/cpp/src/traversal/tsp.cu +++ b/cpp/src/traversal/tsp.cu @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2014-2020, Texas State University. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cpp/src/traversal/tsp.hpp b/cpp/src/traversal/tsp.hpp index 6073f46ab28..f052462156f 100644 --- a/cpp/src/traversal/tsp.hpp +++ b/cpp/src/traversal/tsp.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2014-2020, Texas State University. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cpp/src/traversal/tsp_solver.hpp b/cpp/src/traversal/tsp_solver.hpp index 5fb3ff1d449..9d36357046f 100644 --- a/cpp/src/traversal/tsp_solver.hpp +++ b/cpp/src/traversal/tsp_solver.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2014-2020, Texas State University. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cpp/src/traversal/tsp_utils.hpp b/cpp/src/traversal/tsp_utils.hpp index 48a3e702f09..eab5c09eb2f 100644 --- a/cpp/src/traversal/tsp_utils.hpp +++ b/cpp/src/traversal/tsp_utils.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2014-2020, Texas State University. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/thirdparty/LICENSES/LICENSE.texas_state_university b/thirdparty/LICENSES/LICENSE.texas_state_university new file mode 100644 index 00000000000..7862557ac87 --- /dev/null +++ b/thirdparty/LICENSES/LICENSE.texas_state_university @@ -0,0 +1,24 @@ +Copyright (c) 2014-2020, Texas State University. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Texas State University nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL TEXAS STATE UNIVERSITY BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From fa6f0f1581e072fdcfdeb43a8a8db1a7c698a266 Mon Sep 17 00:00:00 2001 From: Rick Ratzel <3039903+rlratzel@users.noreply.github.com> Date: Thu, 16 Sep 2021 10:49:53 -0500 Subject: [PATCH 3/3] back porting ktruss CUDA 11.4 guard to 21.08 (#1813) --- python/cugraph/community/__init__.py | 39 ++------------------- python/cugraph/community/ktruss_subgraph.py | 27 ++++++++------ 2 files changed, 19 insertions(+), 47 deletions(-) diff --git a/python/cugraph/community/__init__.py b/python/cugraph/community/__init__.py index e8bea0cbaa0..9cc92637e20 100644 --- a/python/cugraph/community/__init__.py +++ b/python/cugraph/community/__init__.py @@ -23,42 +23,7 @@ ) from cugraph.community.subgraph_extraction import subgraph from cugraph.community.triangle_count import triangles +from cugraph.community.ktruss_subgraph import ktruss_subgraph +from cugraph.community.ktruss_subgraph import k_truss from cugraph.community.egonet import ego_graph from cugraph.community.egonet import batched_ego_graphs - -# FIXME: special case for ktruss on CUDA 11.4: an 11.4 bug causes ktruss to -# crash in that environment. Allow ktruss to import on non-11.4 systems, but -# replace ktruss with a __UnsupportedModule instance, which lazily raises an -# exception when referenced. -from numba import cuda -try: - __cuda_version = cuda.runtime.get_version() -except cuda.cudadrv.runtime.CudaRuntimeAPIError: - __cuda_version = "n/a" - -__ktruss_unsupported_cuda_version = (11, 4) - -class __UnsupportedModule: - def __init__(self, exception): - self.__exception = exception - - def __getattr__(self, attr): - raise self.__exception - - def __call__(self, *args, **kwargs): - raise self.__exception - - -if __cuda_version != __ktruss_unsupported_cuda_version: - from cugraph.community.ktruss_subgraph import ktruss_subgraph - from cugraph.community.ktruss_subgraph import k_truss -else: - __kuvs = ".".join([str(n) for n in __ktruss_unsupported_cuda_version]) - k_truss = __UnsupportedModule( - NotImplementedError("k_truss is not currently supported in CUDA" - f" {__kuvs} environments.") - ) - ktruss_subgraph = __UnsupportedModule( - NotImplementedError("ktruss_subgraph is not currently supported in CUDA" - f" {__kuvs} environments.") - ) diff --git a/python/cugraph/community/ktruss_subgraph.py b/python/cugraph/community/ktruss_subgraph.py index c80f65c1608..afa7d66d31d 100644 --- a/python/cugraph/community/ktruss_subgraph.py +++ b/python/cugraph/community/ktruss_subgraph.py @@ -16,21 +16,24 @@ from cugraph.utilities import check_nx_graph from cugraph.utilities import cugraph_to_nx +from numba import cuda + + # FIXME: special case for ktruss on CUDA 11.4: an 11.4 bug causes ktruss to # crash in that environment. Allow ktruss to import on non-11.4 systems, but # raise an exception if ktruss is directly imported on 11.4. -from numba import cuda -try: - __cuda_version = cuda.runtime.get_version() -except cuda.cudadrv.runtime.CudaRuntimeAPIError: - __cuda_version = "n/a" +def _ensure_compatible_cuda_version(): + try: + cuda_version = cuda.runtime.get_version() + except cuda.cudadrv.runtime.CudaRuntimeAPIError: + cuda_version = "n/a" -__ktruss_unsupported_cuda_version = (11, 4) + unsupported_cuda_version = (11, 4) -if __cuda_version == __ktruss_unsupported_cuda_version: - __kuvs = ".".join([str(n) for n in __ktruss_unsupported_cuda_version]) - raise NotImplementedError("k_truss is not currently supported in CUDA" - f" {__kuvs} environments.") + if cuda_version == unsupported_cuda_version: + ver_string = ".".join([str(n) for n in unsupported_cuda_version]) + raise NotImplementedError("k_truss is not currently supported in CUDA" + f" {ver_string} environments.") def k_truss(G, k): @@ -62,6 +65,8 @@ def k_truss(G, k): The networkx graph will NOT have all attributes copied over """ + _ensure_compatible_cuda_version() + G, isNx = check_nx_graph(G) if isNx is True: @@ -137,6 +142,8 @@ def ktruss_subgraph(G, k, use_weights=True): >>> k_subgraph = cugraph.ktruss_subgraph(G, 3) """ + _ensure_compatible_cuda_version() + KTrussSubgraph = Graph() if type(G) is not Graph: raise Exception("input graph must be undirected")