Dependency update (#5308)

* Update cutlass 3.3.0 -> 3.4.0 * Update rapidjson * Update black 23.12.1 -> 24.1.1 * Reformat Python code with the new black * Update libjpeg-turbo 2.1.5.1 -> 3.0.2 in conda recipe * Use most recent DALI_DEPS_VERSION --------- Signed-off-by: Kamil Tokarski <[email protected]>
NVIDIA · Feb 7, 2024 · be1c80c · be1c80c
1 parent 5f6605d
commit be1c80c
Show file tree

Hide file tree

Showing 23 changed files with 58 additions and 61 deletions.
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -17,7 +17,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: '3.10'
-      - run: pip install  flake8 "black[jupyter]"==23.12.1
+      - run: pip install  flake8 "black[jupyter]"==24.1.1
       - run: black --check --verbose ${{ env.PYTHON_LINT_PATHS }} ${{ env.AUTOGRAPH_LINT_PATHS }}
       - run: flake8 --config=.flake8 ${{ env.PYTHON_LINT_PATHS }}
       - run: flake8 --config=.flake8.ag ${{ env.AUTOGRAPH_LINT_PATHS }}

diff --git a/DALI_DEPS_VERSION b/DALI_DEPS_VERSION
@@ -1 +1 @@
-1e85021cae11d99e9432c013f8e19155b08ff2e5
+df4a84e20b4133442f3f2e03d6f80422d2d81f35
diff --git a/conda/dali_python_bindings/recipe/meta.yaml b/conda/dali_python_bindings/recipe/meta.yaml
@@ -1,5 +1,5 @@
 # (C) Copyright IBM Corp. 2019. All Rights Reserved.
-# Copyright (c) 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright (c) 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -72,8 +72,8 @@ requirements:
     - astunparse >=1.6.0
     - gast >=0.3.3
     - dm-tree >=0.1.8
-    - black =23.12.1
-    - black-jupyter =23.12.1
+    - black =24.1.1
+    - black-jupyter =24.1.1
     - nvidia-dali-core{% if environ.get('NVIDIA_DALI_BUILD_FLAVOR', '')|length %}{{"-" + environ.get('NVIDIA_DALI_BUILD_FLAVOR', '')}}{% endif %}-cuda{{ environ.get('CUDA_VERSION', '') | replace(".","") }} ={{ environ.get('DALI_CONDA_BUILD_VERSION', '') }}
   host:
     - dali-opencv

diff --git a/conda/third_party/jpeg_turbo/recipe/meta.yaml b/conda/third_party/jpeg_turbo/recipe/meta.yaml
@@ -20,7 +20,7 @@
 
   source:
     git_url: https://github.com/libjpeg-turbo/libjpeg-turbo
-    git_rev: 2.1.5.1
+    git_rev: 3.0.2
 
   build:
     number: 0

diff --git a/dali/python/nvidia/dali/_multiproc/pool.py b/dali/python/nvidia/dali/_multiproc/pool.py
@@ -68,7 +68,6 @@
 
 
 class ShmChunkManager:
-
     """Two dimensional buffer of shared memory chunks (queue_depth X num_minibatches),
     chunks can be accessed either by providing two coordinates or via shm chunk's unique id.
     Each ExternalSource callback gets its own buffer, first dimension is cycled

diff --git a/dali/python/nvidia/dali/_multiproc/shared_queue.py b/dali/python/nvidia/dali/_multiproc/shared_queue.py
@@ -25,7 +25,6 @@ class QueueMeta(Structure):
 
 
 class ShmQueue:
-
     """
     Simple fixed capacity shared memory queue of fixed size messages.
     Writing to a full queue fails, attempt to get from an empty queue blocks until data is
@@ -184,7 +183,6 @@ def get(self, num_samples=1, predicate=None) -> Optional[List[MSG_CLASS]]:
 
 
 class Dispatcher:
-
     """Wrapper around the queue that enables writing to the queue in a separate thread, just in
     case a writing process would have to wait too long for a lock on the queue when multiple
     readers pop the items one by one."""

diff --git a/dali/python/nvidia/dali/_multiproc/struct_message.py b/dali/python/nvidia/dali/_multiproc/struct_message.py
@@ -17,7 +17,6 @@
 
 
 class Structure:
-
     """
     Utility around Python `struct` module (https://docs.python.org/3.6/library/struct.html)
      that allows to access and modify `_fields` like an ordinary object attributes

diff --git a/dali/python/nvidia/dali/_typing.py b/dali/python/nvidia/dali/_typing.py
@@ -25,8 +25,7 @@ class ArrayInterface(Protocol):
     to its operators. Such parameter would be broadcast for all samples in the batch.
     """
 
-    def __array__(self) -> Any:
-        ...
+    def __array__(self) -> Any: ...
 
 
 class CudaArrayInterface(Protocol):
@@ -37,8 +36,7 @@ class CudaArrayInterface(Protocol):
     """
 
     @property
-    def __cuda_array_interface__(self) -> Any:
-        ...
+    def __cuda_array_interface__(self) -> Any: ...
 
 
 TensorLikeIn: TypeAlias = Union[ArrayInterface, Sequence[int], Sequence[float], int, float]

diff --git a/dali/python/nvidia/dali/pipeline.py b/dali/python/nvidia/dali/pipeline.py
@@ -1780,8 +1780,7 @@ def _generate_graph(pipe, func, fn_args, fn_kwargs):
 @overload
 def pipeline_def(
     __func: Callable[..., Union[DataNode, Tuple[DataNode, ...]]]
-) -> Callable[..., Pipeline]:
-    ...
+) -> Callable[..., Pipeline]: ...
 
 
 # Decorator with arguments
@@ -1808,8 +1807,7 @@ def pipeline_def(
     py_callback_pickler: Optional[Any] = None,
     output_dtype: Union[types.DALIDataType, Tuple[types.DALIDataType, ...], None] = None,
     output_ndim: Union[int, Tuple[int, ...], None] = None,
-) -> Callable[[Callable[..., Union[DataNode, Tuple[DataNode, ...]]]], Callable[..., Pipeline]]:
-    ...
+) -> Callable[[Callable[..., Union[DataNode, Tuple[DataNode, ...]]]], Callable[..., Pipeline]]: ...
 
 
 # Implementation

diff --git a/dali/test/python/operator_1/test_coord_transform.py b/dali/test/python/operator_1/test_coord_transform.py
@@ -94,9 +94,11 @@ def _run_test(device, batch_size, out_dim, in_dim, in_dtype, out_dtype, M_kind,
             MT = 0
 
         M, T, MT = (
-            x
-            if isinstance(x, dali.data_node.DataNode)
-            else dali.types.Constant(x, dtype=dali.types.FLOAT)
+            (
+                x
+                if isinstance(x, dali.data_node.DataNode)
+                else dali.types.Constant(x, dtype=dali.types.FLOAT)
+            )
             for x in (M, T, MT)
         )
 

diff --git a/dali/test/python/operator_1/test_crop_mirror_normalize.py b/dali/test/python/operator_1/test_crop_mirror_normalize.py
@@ -909,9 +909,11 @@ def pipe():
         outs = p.run()
         for s in range(batch_size):
             out, image_like, mean, std = [
-                np.array(o[s].as_cpu())
-                if isinstance(o, dali.backend_impl.TensorListGPU)
-                else np.array(o[s])
+                (
+                    np.array(o[s].as_cpu())
+                    if isinstance(o, dali.backend_impl.TensorListGPU)
+                    else np.array(o[s])
+                )
                 for o in outs
             ]
         ref_scale = scale or 1.0

diff --git a/dali/test/python/operator_1/test_erase.py b/dali/test/python/operator_1/test_erase.py
@@ -129,9 +129,9 @@ def erase_func(anchor, shape, axis_names, axes, layout, fill_value, image):
         if len(start) == 3:
             image[start[0] : end[0], start[1] : end[1], start[2] : end[2]] = fill_value
         elif len(start) == 4:
-            image[
-                start[0] : end[0], start[1] : end[1], start[2] : end[2], start[3] : end[3]
-            ] = fill_value
+            image[start[0] : end[0], start[1] : end[1], start[2] : end[2], start[3] : end[3]] = (
+                fill_value
+            )
         else:
             assert False
     return image

diff --git a/dali/test/python/operator_2/test_rotate.py b/dali/test/python/operator_2/test_rotate.py
@@ -310,9 +310,9 @@ def iter_by_groups():
         assert axis_batch is None or len(axis_batch) == len(angle_batch)
         offset = 0
         for group in unfolded_extents:
-            yield input_batch[offset : offset + group], angle_batch[
-                offset : offset + group
-            ], None if axis_batch is None else axis_batch[offset : offset + group]
+            yield input_batch[offset : offset + group], angle_batch[offset : offset + group], (
+                None if axis_batch is None else axis_batch[offset : offset + group]
+            )
             offset += group
 
     sequence_output_shape = [

diff --git a/dali/test/python/test_dali_tf_dataset_shape.py b/dali/test/python/test_dali_tf_dataset_shape.py
@@ -150,9 +150,7 @@ def test_batch_N_valid_shapes():
     # Omitted batch of size `1`
     output_shape = (200, 200, 3)
     for i in range(2 ** len(output_shape)):
-        noned_shape = tuple(
-            (dim if i & (2**idx) else None) for idx, dim in enumerate(output_shape)
-        )
+        noned_shape = tuple((dim if i & (2**idx) else None) for idx, dim in enumerate(output_shape))
         yield dali_pipe_batch_N, noned_shape, tf.uint8, 1
 
 

diff --git a/dali/test/python/test_external_source_parallel_custom_serialization.py b/dali/test/python/test_external_source_parallel_custom_serialization.py
@@ -360,9 +360,9 @@ def _test_global_function_pickled_by_reference(name, py_callback_pickler):
 def _test_pickle_by_value_decorator_on_global_function(name, py_callback_pickler):
     # modify callback name so that an attempt to pickle by reference,
     # which is default Python behavior, would fail
-    _simple_callback_by_value.__name__ = (
-        _simple_callback_by_value.__qualname__
-    ) = "simple_callback_by_value"
+    _simple_callback_by_value.__name__ = _simple_callback_by_value.__qualname__ = (
+        "simple_callback_by_value"
+    )
     _create_and_compare_simple_pipelines(
         _simple_callback_by_value, py_callback_pickler, batch_size=4, py_num_workers=2
     )

diff --git a/dali/test/python/webdataset_base.py b/dali/test/python/webdataset_base.py
@@ -84,20 +84,22 @@ def file_reader_pipeline(
         exts = [exts]
 
     return tuple(
-        readers.file(
-            files=filter_ext(files, ext),
-            dont_use_mmap=dont_use_mmap,
-            prefetch_queue_depth=1,
-            num_shards=num_shards,
-            shard_id=shard_id,
-            stick_to_shard=stick_to_shard,
-            skip_cached_images=skip_cached_images,
-            pad_last_batch=pad_last_batch,
-            lazy_init=lazy_init,
-            read_ahead=read_ahead,
-        )[0]
-        if type(ext) in {str, set}
-        else ext
+        (
+            readers.file(
+                files=filter_ext(files, ext),
+                dont_use_mmap=dont_use_mmap,
+                prefetch_queue_depth=1,
+                num_shards=num_shards,
+                shard_id=shard_id,
+                stick_to_shard=stick_to_shard,
+                skip_cached_images=skip_cached_images,
+                pad_last_batch=pad_last_batch,
+                lazy_init=lazy_init,
+                read_ahead=read_ahead,
+            )[0]
+            if type(ext) in {str, set}
+            else ext
+        )
         for ext in exts
     )
 

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -20,7 +20,7 @@ ENV PYVER=${PYVER} \
 RUN ln -s /opt/python/cp${PYV}* /opt/python/v
 
 # install Python bindings and patch it to use the clang we have here
-RUN pip install future setuptools wheel clang==14.0 flake8 astunparse gast dm-tree "black[jupyter]"==23.12.1 && \
+RUN pip install future setuptools wheel clang==14.0 flake8 astunparse gast dm-tree "black[jupyter]"==24.1.1 && \
     PY_CLANG_PATH=$(echo $(pip show clang) | sed 's/.*Location: \(.*\) Requires.*/\1/')/clang/cindex.py && \
     LIBCLANG_PATH=/usr/local/lib/libclang.so && \
     sed -i "s|library_file = None|library_file = \"${LIBCLANG_PATH}\"|" ${PY_CLANG_PATH} && \

diff --git a/docker/Dockerfile.build.aarch64-linux b/docker/Dockerfile.build.aarch64-linux
@@ -50,7 +50,7 @@ RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/
         curl -O https://bootstrap.pypa.io/get-pip.py; \
     fi && python3 get-pip.py && rm get-pip.py && \
     # decouple libclang and clang installation so libclang changes are not overriden by clang
-    pip install clang==14.0 && pip install libclang==14.0.1 flake8 "black[jupyter]"==23.12.1 && \
+    pip install clang==14.0 && pip install libclang==14.0.1 flake8 "black[jupyter]"==24.1.1 && \
     rm -rf /root/.cache/pip/ && \
     cd /tmp && git clone https://github.com/NixOS/patchelf && cd patchelf && \
     ./bootstrap.sh && ./configure --prefix=/usr/ && make -j install && cd / && rm -rf /tmp/patchelf && \

diff --git a/docs/examples/frameworks/mxnet/demo/symbols/vgg16.py b/docs/examples/frameworks/mxnet/demo/symbols/vgg16.py
@@ -5,6 +5,7 @@
 
 This implements Variant D from the paper.
 """
+
 import mxnet as mx
 
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.black]
 line-length = 100
 target-version = ['py38', 'py39', 'py310', 'py311']
-required-version = '23'
+required-version = '24'
 workers = 32
 extend-exclude = ".*/docs/examples/use_cases/.*"
diff --git a/third_party/README.rst b/third_party/README.rst
@@ -49,8 +49,8 @@ This part of the repository contains extra dependencies required to build DALI,
 
 .. |cutlass| replace:: CUTLASS
 .. _cutlass: https://github.com/NVIDIA/cutlass
-.. |cutlassver| replace:: 3.3.0
-.. _cutlassver: https://github.com/NVIDIA/cutlass/tree/v3.3.0
+.. |cutlassver| replace:: 3.4.0
+.. _cutlassver: https://github.com/NVIDIA/cutlass/tree/v3.4.0
 .. |cutlasslic| replace:: BSD 3-Clause License
 .. _cutlasslic: https://github.com/NVIDIA/cutlass/blob/master/LICENSE.txt
 
@@ -84,14 +84,14 @@ This part of the repository contains extra dependencies required to build DALI,
 
 .. |rapidjson| replace:: RapidJSON
 .. _rapidjson: https://github.com/Tencent/rapidjson
-.. |rapidjsonver| replace:: Top-of-tree (Sep 28, 2023)
-.. _rapidjsonver: https://github.com/Tencent/rapidjson/commit/f9d53419e912910fd8fa57d5705fa41425428c35
+.. |rapidjsonver| replace:: Top-of-tree (Dec 6, 2023)
+.. _rapidjsonver: https://github.com/Tencent/rapidjson/commit/6089180ecb704cb2b136777798fa1be303618975
 .. |rapidjsonlic| replace:: MIT License, BSD 3-Clause License, JSON License
 .. _rapidjsonlic: https://github.com/Tencent/rapidjson/blob/master/license.txt
 
 .. |black| replace:: black
 .. _black: https://github.com/psf/black
-.. |blackver| replace:: 23.12.1
-.. _blackver: https://pypi.org/project/black/23.12.1/
+.. |blackver| replace:: 24.1.1
+.. _blackver: https://pypi.org/project/black/24.1.1/
 .. |blacklic| replace:: MIT License
 .. _blacklic: https://github.com/psf/black/blob/main/LICENSE
diff --git a/third_party/cutlass b/third_party/cutlass
diff --git a/third_party/rapidjson b/third_party/rapidjson