From d93d90dcaee808525b83f88f9ce39b55ede06d96 Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Mon, 17 Jun 2024 13:46:21 -0600 Subject: [PATCH 1/9] Add ruff setup --- .pre-commit-config.yaml | 26 ++++------------- pyproject.toml | 65 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 63 insertions(+), 28 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 656fb790..381cb30d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,25 +6,9 @@ repos: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace - - repo: 'https://github.com/asottile/pyupgrade' - rev: v3.15.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.8 hooks: - - id: pyupgrade - args: - - '--py37-plus' - - repo: 'https://github.com/PyCQA/isort' - rev: 5.12.0 - hooks: - - id: isort - - repo: 'https://github.com/psf/black' - rev: 23.11.0 - hooks: - - id: black - - repo: 'https://github.com/pycqa/flake8' - rev: 6.1.0 - hooks: - - id: flake8 - - repo: 'https://github.com/codespell-project/codespell' - rev: v2.1.0 - hooks: - - id: codespell + - id: ruff + args: [ --fix ] + - id: ruff-format diff --git a/pyproject.toml b/pyproject.toml index 0c1026fe..c93b2674 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,12 +3,63 @@ markers = [ "slow: marks tests as slow", ] -[tool.isort] -profile = "black" -line_length = 79 -honor_noqa = true -src_paths = ["s3transfer", "tests"] +[tool.ruff] +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] -[tool.black] +# Format same as Black. line-length = 79 -skip_string_normalization = true +indent-width = 4 + +target-version = "py38" + +[tool.ruff.lint] +# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. +# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or +# McCabe complexity (`C901`) by default. +select = ["E4", "E7", "E9", "F", "UP"] +ignore = [] + +# Allow fix for all enabled rules (when `--fix`) is provided. +fixable = ["ALL"] +unfixable = [] + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[tool.ruff.format] +# Like Black, use double quotes for strings, spaces for indents +# and trailing commas. +quote-style = "preserve" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" + +docstring-code-format = false +docstring-code-line-length = "dynamic" From 425c9040740ca57ffcc28d5b3f1e117717c37d9e Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Mon, 17 Jun 2024 13:49:46 -0600 Subject: [PATCH 2/9] Run ruff over codebase --- s3transfer/__init__.py | 1 + s3transfer/bandwidth.py | 4 +--- s3transfer/download.py | 4 +--- s3transfer/futures.py | 8 ++------ s3transfer/processpool.py | 1 + s3transfer/subscribers.py | 2 +- s3transfer/tasks.py | 7 +++---- s3transfer/upload.py | 4 +--- s3transfer/utils.py | 4 +--- scripts/new-change | 5 ++--- scripts/performance/benchmark | 1 + scripts/performance/benchmark-download | 1 + scripts/performance/benchmark-upload | 1 + scripts/performance/download-file | 1 + scripts/performance/processpool-download | 1 + scripts/performance/summarize | 1 + scripts/performance/upload-file | 1 + scripts/stress/timeout | 1 + tests/__init__.py | 4 +--- tests/functional/test_copy.py | 12 ++++++------ tests/functional/test_download.py | 6 +++--- tests/integration/test_download.py | 8 ++------ tests/integration/test_processpool.py | 8 ++------ tests/integration/test_s3transfer.py | 4 +--- tests/integration/test_upload.py | 8 ++------ tests/unit/test_compat.py | 2 +- tests/unit/test_futures.py | 4 +--- tests/unit/test_s3transfer.py | 9 ++++----- tests/unit/test_utils.py | 2 +- 29 files changed, 46 insertions(+), 69 deletions(-) diff --git a/s3transfer/__init__.py b/s3transfer/__init__.py index 4ada4a89..7a6af1fc 100644 --- a/s3transfer/__init__.py +++ b/s3transfer/__init__.py @@ -123,6 +123,7 @@ def __call__(self, bytes_amount): """ + import concurrent.futures import functools import logging diff --git a/s3transfer/bandwidth.py b/s3transfer/bandwidth.py index 9bac5885..9301c8e3 100644 --- a/s3transfer/bandwidth.py +++ b/s3transfer/bandwidth.py @@ -30,9 +30,7 @@ def __init__(self, requested_amt, retry_time): """ self.requested_amt = requested_amt self.retry_time = retry_time - msg = 'Request amount {} exceeded the amount available. Retry in {}'.format( - requested_amt, retry_time - ) + msg = f'Request amount {requested_amt} exceeded the amount available. Retry in {retry_time}' super().__init__(msg) diff --git a/s3transfer/download.py b/s3transfer/download.py index dc8980d4..e1a9cf0e 100644 --- a/s3transfer/download.py +++ b/s3transfer/download.py @@ -307,9 +307,7 @@ def _get_download_output_manager_cls(self, transfer_future, osutil): if download_manager_cls.is_compatible(fileobj, osutil): return download_manager_cls raise RuntimeError( - 'Output {} of type: {} is not supported.'.format( - fileobj, type(fileobj) - ) + f'Output {fileobj} of type: {type(fileobj)} is not supported.' ) def _submit( diff --git a/s3transfer/futures.py b/s3transfer/futures.py index e8282916..540f5cc6 100644 --- a/s3transfer/futures.py +++ b/s3transfer/futures.py @@ -175,9 +175,7 @@ def __init__(self, transfer_id=None): self._failure_cleanups_lock = threading.Lock() def __repr__(self): - return '{}(transfer_id={})'.format( - self.__class__.__name__, self.transfer_id - ) + return f'{self.__class__.__name__}(transfer_id={self.transfer_id})' @property def exception(self): @@ -316,9 +314,7 @@ def submit(self, executor, task, tag=None): :returns: A future representing the submitted task """ logger.debug( - "Submitting task {} to executor {} for transfer request: {}.".format( - task, executor, self.transfer_id - ) + f"Submitting task {task} to executor {executor} for transfer request: {self.transfer_id}." ) future = executor.submit(task, tag=tag) # Add this created future to the list of associated future just diff --git a/s3transfer/processpool.py b/s3transfer/processpool.py index 017eeb44..318f1e7f 100644 --- a/s3transfer/processpool.py +++ b/s3transfer/processpool.py @@ -192,6 +192,7 @@ are using ``us-west-2`` as their region. """ + import collections import contextlib import logging diff --git a/s3transfer/subscribers.py b/s3transfer/subscribers.py index 473d5d94..eeed5e43 100644 --- a/s3transfer/subscribers.py +++ b/s3transfer/subscribers.py @@ -30,7 +30,7 @@ def __new__(cls, *args, **kwargs): return super().__new__(cls) @classmethod - @lru_cache() + @lru_cache def _validate_subscriber_methods(cls): for subscriber_type in cls.VALID_SUBSCRIBER_TYPES: subscriber_method = getattr(cls, 'on_' + subscriber_type) diff --git a/s3transfer/tasks.py b/s3transfer/tasks.py index 1bad9812..4917dfc9 100644 --- a/s3transfer/tasks.py +++ b/s3transfer/tasks.py @@ -96,10 +96,9 @@ def __repr__(self): main_kwargs_to_display = self._get_kwargs_with_params_to_include( self._main_kwargs, params_to_display ) - return '{}(transfer_id={}, {})'.format( - self.__class__.__name__, - self._transfer_coordinator.transfer_id, - main_kwargs_to_display, + return ( + f'{self.__class__.__name__}(transfer_id=' + f'{self._transfer_coordinator.transfer_id}, {main_kwargs_to_display})' ) @property diff --git a/s3transfer/upload.py b/s3transfer/upload.py index c834c52a..0347857b 100644 --- a/s3transfer/upload.py +++ b/s3transfer/upload.py @@ -550,9 +550,7 @@ def _get_upload_input_manager_cls(self, transfer_future): if upload_manager_cls.is_compatible(fileobj): return upload_manager_cls raise RuntimeError( - 'Input {} of type: {} is not supported.'.format( - fileobj, type(fileobj) - ) + f'Input {fileobj} of type: {type(fileobj)} is not supported.' ) def _submit( diff --git a/s3transfer/utils.py b/s3transfer/utils.py index ef171f54..62ecf39c 100644 --- a/s3transfer/utils.py +++ b/s3transfer/utils.py @@ -191,9 +191,7 @@ def __init__(self, func, *args, **kwargs): self._kwargs = kwargs def __repr__(self): - return 'Function: {} with args {} and kwargs {}'.format( - self._func, self._args, self._kwargs - ) + return f'Function: {self._func} with args {self._args} and kwargs {self._kwargs}' def __call__(self): return self._func(*self._args, **self._kwargs) diff --git a/scripts/new-change b/scripts/new-change index 78c4fa9f..bdf898fa 100755 --- a/scripts/new-change +++ b/scripts/new-change @@ -36,6 +36,7 @@ You can then use the ``scripts/gen-changelog`` to generate the CHANGELOG.rst file. """ + import argparse import json import os @@ -132,9 +133,7 @@ def replace_issue_references(parsed, repo_name): def linkify(match): number = match.group()[1:] - return '`{} `__'.format( - match.group(), repo_name, number - ) + return f'`{match.group()} `__' new_description = re.sub(r'#\d+', linkify, description) parsed['description'] = new_description diff --git a/scripts/performance/benchmark b/scripts/performance/benchmark index 30152ade..af8dbb21 100755 --- a/scripts/performance/benchmark +++ b/scripts/performance/benchmark @@ -18,6 +18,7 @@ To use the script, run:: If no ``--output-file`` was provided, the data will be saved to ``performance.csv`` """ + import argparse import os import subprocess diff --git a/scripts/performance/benchmark-download b/scripts/performance/benchmark-download index 932fd51f..30fdba1c 100755 --- a/scripts/performance/benchmark-download +++ b/scripts/performance/benchmark-download @@ -19,6 +19,7 @@ To benchmark with your own s3 key: --s3-bucket mybucket """ + import argparse import os import shutil diff --git a/scripts/performance/benchmark-upload b/scripts/performance/benchmark-upload index bbaf6dfa..64680535 100755 --- a/scripts/performance/benchmark-upload +++ b/scripts/performance/benchmark-upload @@ -19,6 +19,7 @@ To benchmark with your own local file:: --s3-bucket mybucket """ + import argparse import os import shutil diff --git a/scripts/performance/download-file b/scripts/performance/download-file index 4a7d86fe..64975d09 100755 --- a/scripts/performance/download-file +++ b/scripts/performance/download-file @@ -14,6 +14,7 @@ To download a file:: --s3-bucket mybucket --s3-key mykey """ + import argparse from botocore.session import get_session diff --git a/scripts/performance/processpool-download b/scripts/performance/processpool-download index 1b175b2f..28295276 100755 --- a/scripts/performance/processpool-download +++ b/scripts/performance/processpool-download @@ -16,6 +16,7 @@ To download a prefix recursively to a directory:: ./proccesspool-download -d mydirname -b mybucket -p myprefix/ """ + import argparse import os diff --git a/scripts/performance/summarize b/scripts/performance/summarize index dafbca2e..fb3a5cc9 100755 --- a/scripts/performance/summarize +++ b/scripts/performance/summarize @@ -63,6 +63,7 @@ summary as JSON instead of a pretty printed table:: } """ + import argparse import csv import json diff --git a/scripts/performance/upload-file b/scripts/performance/upload-file index 3b0abc8a..d61a48de 100755 --- a/scripts/performance/upload-file +++ b/scripts/performance/upload-file @@ -14,6 +14,7 @@ To upload a file:: --s3-bucket mybucket --s3-key mykey """ + import argparse from botocore.session import get_session diff --git a/scripts/stress/timeout b/scripts/stress/timeout index 3323b4aa..fa5bba52 100755 --- a/scripts/stress/timeout +++ b/scripts/stress/timeout @@ -13,6 +13,7 @@ To use the script, run:: ./timeout "./my-script-to-run" --timeout-after 5 """ + import argparse import os import subprocess diff --git a/tests/__init__.py b/tests/__init__.py index 03590fef..b5c50327 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -72,9 +72,7 @@ def assert_files_equal(first, second): second_md5 = md5_checksum(second) if first_md5 != second_md5: raise AssertionError( - "Files are not equal: {}(md5={}) != {}(md5={})".format( - first, first_md5, second, second_md5 - ) + f"Files are not equal: {first}(md5={first_md5}) != {second}(md5={second_md5})" ) diff --git a/tests/functional/test_copy.py b/tests/functional/test_copy.py index 9638cab2..d13755d8 100644 --- a/tests/functional/test_copy.py +++ b/tests/functional/test_copy.py @@ -96,9 +96,9 @@ def add_successful_copy_responses( # Add the expected create multipart upload params. if expected_create_mpu_params: - stubbed_responses[0][ - 'expected_params' - ] = expected_create_mpu_params + stubbed_responses[0]['expected_params'] = ( + expected_create_mpu_params + ) # Add any expected copy parameters. if expected_copy_params: @@ -110,9 +110,9 @@ def add_successful_copy_responses( # Add the expected complete multipart upload params. if expected_complete_mpu_params: - stubbed_responses[-1][ - 'expected_params' - ] = expected_complete_mpu_params + stubbed_responses[-1]['expected_params'] = ( + expected_complete_mpu_params + ) # Add the responses to the stubber. for stubbed_response in stubbed_responses: diff --git a/tests/functional/test_download.py b/tests/functional/test_download.py index f458721d..a36a2356 100644 --- a/tests/functional/test_download.py +++ b/tests/functional/test_download.py @@ -116,9 +116,9 @@ def add_successful_get_object_responses( expected_params ) if expected_ranges: - stubbed_response['expected_params'][ - 'Range' - ] = expected_ranges[i] + stubbed_response['expected_params']['Range'] = ( + expected_ranges[i] + ) self.stubber.add_response(**stubbed_response) def add_n_retryable_get_object_responses(self, n, num_reads=0): diff --git a/tests/integration/test_download.py b/tests/integration/test_download.py index 6a07f933..9561f61d 100644 --- a/tests/integration/test_download.py +++ b/tests/integration/test_download.py @@ -115,9 +115,7 @@ def test_large_download_exits_quicky_on_exception(self): self.assertLess( actual_time_to_exit, max_allowed_exit_time, - "Failed to exit under {}. Instead exited in {}.".format( - max_allowed_exit_time, actual_time_to_exit - ), + f"Failed to exit under {max_allowed_exit_time}. Instead exited in {actual_time_to_exit}.", ) # Make sure the future was cancelled because of the KeyboardInterrupt @@ -174,9 +172,7 @@ def test_many_files_exits_quicky_on_exception(self): self.assertLess( end_time - start_time, max_allowed_exit_time, - "Failed to exit under {}. Instead exited in {}.".format( - max_allowed_exit_time, end_time - start_time - ), + f"Failed to exit under {max_allowed_exit_time}. Instead exited in {end_time - start_time}.", ) # Make sure at least one of the futures got cancelled diff --git a/tests/integration/test_processpool.py b/tests/integration/test_processpool.py index d0899b1b..4287f3b1 100644 --- a/tests/integration/test_processpool.py +++ b/tests/integration/test_processpool.py @@ -96,9 +96,7 @@ def test_large_download_exits_quickly_on_exception(self): self.assertLess( end_time - start_time, max_allowed_exit_time, - "Failed to exit under {}. Instead exited in {}.".format( - max_allowed_exit_time, end_time - start_time - ), + f"Failed to exit under {max_allowed_exit_time}. Instead exited in {end_time - start_time}.", ) # Make sure the actual file and the temporary do not exist @@ -138,9 +136,7 @@ def test_many_files_exits_quickly_on_exception(self): self.assertLess( end_time - start_time, max_allowed_exit_time, - "Failed to exit under {}. Instead exited in {}.".format( - max_allowed_exit_time, end_time - start_time - ), + f"Failed to exit under {max_allowed_exit_time}. Instead exited in {end_time - start_time}.", ) # For the transfer that did get cancelled, make sure the temporary diff --git a/tests/integration/test_s3transfer.py b/tests/integration/test_s3transfer.py index 95decb4d..f710dd45 100644 --- a/tests/integration/test_s3transfer.py +++ b/tests/integration/test_s3transfer.py @@ -31,9 +31,7 @@ def assert_files_equal(first, second): second_md5 = md5_checksum(second) if first_md5 != second_md5: raise AssertionError( - "Files are not equal: {}(md5={}) != {}(md5={})".format( - first, first_md5, second, second_md5 - ) + f"Files are not equal: {first}(md5={first_md5}) != {second}(md5={second_md5})" ) diff --git a/tests/integration/test_upload.py b/tests/integration/test_upload.py index ef366de5..cf6a22f8 100644 --- a/tests/integration/test_upload.py +++ b/tests/integration/test_upload.py @@ -101,9 +101,7 @@ def test_large_upload_exits_quicky_on_exception(self): self.assertLess( actual_time_to_exit, max_allowed_exit_time, - "Failed to exit under {}. Instead exited in {}.".format( - max_allowed_exit_time, actual_time_to_exit - ), + f"Failed to exit under {max_allowed_exit_time}. Instead exited in {actual_time_to_exit}.", ) try: @@ -163,9 +161,7 @@ def test_many_files_exits_quicky_on_exception(self): self.assertLess( end_time - start_time, max_allowed_exit_time, - "Failed to exit under {}. Instead exited in {}.".format( - max_allowed_exit_time, end_time - start_time - ), + f"Failed to exit under {max_allowed_exit_time}. Instead exited in {end_time - start_time}.", ) # Make sure at least one of the futures got cancelled diff --git a/tests/unit/test_compat.py b/tests/unit/test_compat.py index f40a602a..20f07c74 100644 --- a/tests/unit/test_compat.py +++ b/tests/unit/test_compat.py @@ -57,7 +57,7 @@ def test_non_file_like_obj(self): def test_non_seekable_ioerror(self): # Should return False if IOError is thrown. with open(self.filename, 'w') as f: - self.assertFalse(seekable(ErrorRaisingSeekWrapper(f, IOError()))) + self.assertFalse(seekable(ErrorRaisingSeekWrapper(f, OSError()))) def test_non_seekable_oserror(self): # Should return False if OSError is thrown. diff --git a/tests/unit/test_futures.py b/tests/unit/test_futures.py index ed196a3a..f2a94f23 100644 --- a/tests/unit/test_futures.py +++ b/tests/unit/test_futures.py @@ -489,9 +489,7 @@ def assert_submit_would_not_block(self, task, tag=None, **kwargs): self.executor.submit(task, tag=tag, block=False) except NoResourcesAvailable: self.fail( - 'Task {} should not have been blocked. Caused by:\n{}'.format( - task, traceback.format_exc() - ) + f'Task {task} should not have been blocked. Caused by:\n{traceback.format_exc()}' ) def add_done_callback_to_future(self, future, fn, *args, **kwargs): diff --git a/tests/unit/test_s3transfer.py b/tests/unit/test_s3transfer.py index d9255bf3..65972ec1 100644 --- a/tests/unit/test_s3transfer.py +++ b/tests/unit/test_s3transfer.py @@ -12,7 +12,6 @@ # language governing permissions and limitations under the License. import os import shutil -import socket import tempfile from concurrent import futures from contextlib import closing @@ -434,7 +433,7 @@ def test_retry_on_failures_from_stream_reads(self): response_body = b'foobarbaz' stream_with_errors = mock.Mock() stream_with_errors.read.side_effect = [ - socket.error("fake error"), + OSError("fake error"), response_body, ] client.get_object.return_value = {'Body': stream_with_errors} @@ -469,7 +468,7 @@ def test_exception_raised_on_exceeded_retries(self): client = mock.Mock() response_body = b'foobarbaz' stream_with_errors = mock.Mock() - stream_with_errors.read.side_effect = socket.error("fake error") + stream_with_errors.read.side_effect = OSError("fake error") client.get_object.return_value = {'Body': stream_with_errors} config = TransferConfig(multipart_threshold=4, multipart_chunksize=4) @@ -678,7 +677,7 @@ def test_get_object_stream_is_retried_and_succeeds(self): } self.client.get_object.side_effect = [ # First request fails. - socket.error("fake error"), + OSError("fake error"), # Second succeeds. {'Body': BytesIO(b'foobar')}, ] @@ -696,7 +695,7 @@ def test_get_object_stream_uses_all_retries_and_errors_out(self): # Here we're raising an exception every single time, which # will exhaust our retry count and propagate a # RetriesExceededError. - self.client.get_object.side_effect = socket.error("fake error") + self.client.get_object.side_effect = OSError("fake error") with self.assertRaises(RetriesExceededError): transfer.download_file('bucket', 'key', 'smallfile') diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ea7edf8a..13a079d9 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -329,7 +329,7 @@ def test_allocate(self): @mock.patch('s3transfer.utils.fallocate') def test_allocate_with_io_error(self, mock_fallocate): - mock_fallocate.side_effect = IOError() + mock_fallocate.side_effect = OSError() with self.assertRaises(IOError): OSUtils().allocate(self.filename, 1) self.assertFalse(os.path.exists(self.filename)) From 1c99dc49e2188f007155a3fb91d48d99267ef291 Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Mon, 17 Jun 2024 13:55:19 -0600 Subject: [PATCH 3/9] Add unsafe changes --- s3transfer/__init__.py | 4 ++-- s3transfer/constants.py | 4 ++-- s3transfer/copies.py | 2 +- s3transfer/futures.py | 6 +++--- s3transfer/manager.py | 12 ++++++------ s3transfer/subscribers.py | 7 +++---- s3transfer/tasks.py | 5 +---- s3transfer/utils.py | 15 +++++++-------- scripts/ci/install | 2 +- scripts/performance/benchmark | 2 +- scripts/performance/benchmark-download | 6 +++--- scripts/performance/benchmark-upload | 7 +++---- scripts/performance/summarize | 12 ++++++------ scripts/stress/timeout | 2 +- tests/functional/test_copy.py | 2 +- tests/functional/test_crt.py | 2 +- tests/functional/test_download.py | 4 ++-- tests/functional/test_processpool.py | 2 +- tests/functional/test_upload.py | 4 ++-- tests/integration/test_crt.py | 2 +- tests/integration/test_download.py | 8 ++++---- tests/integration/test_processpool.py | 4 ++-- tests/integration/test_upload.py | 2 +- tests/unit/test_crt.py | 2 +- tests/unit/test_manager.py | 2 +- tests/unit/test_subscribers.py | 2 +- tests/unit/test_upload.py | 2 +- tests/unit/test_utils.py | 4 ++-- 28 files changed, 61 insertions(+), 67 deletions(-) diff --git a/s3transfer/__init__.py b/s3transfer/__init__.py index 7a6af1fc..36626c12 100644 --- a/s3transfer/__init__.py +++ b/s3transfer/__init__.py @@ -814,8 +814,8 @@ def _validate_all_known_args(self, actual, allowed): for kwarg in actual: if kwarg not in allowed: raise ValueError( - "Invalid extra_args key '%s', " - "must be one of: %s" % (kwarg, ', '.join(allowed)) + f"Invalid extra_args key '{kwarg}', " + f"must be one of: {', '.join(allowed)}" ) def _ranged_download( diff --git a/s3transfer/constants.py b/s3transfer/constants.py index 570aa2ea..b07b1d47 100644 --- a/s3transfer/constants.py +++ b/s3transfer/constants.py @@ -26,5 +26,5 @@ 'ExpectedBucketOwner', ] -USER_AGENT = 's3transfer/%s' % s3transfer.__version__ -PROCESS_USER_AGENT = '%s processpool' % USER_AGENT +USER_AGENT = f's3transfer/{s3transfer.__version__}' +PROCESS_USER_AGENT = f'{USER_AGENT} processpool' diff --git a/s3transfer/copies.py b/s3transfer/copies.py index 77deca62..c2ae9ce0 100644 --- a/s3transfer/copies.py +++ b/s3transfer/copies.py @@ -280,7 +280,7 @@ def _get_head_object_request_from_copy_source(self, copy_source): raise TypeError( 'Expecting dictionary formatted: ' '{"Bucket": bucket_name, "Key": key} ' - 'but got %s or type %s.' % (copy_source, type(copy_source)) + f'but got {copy_source} or type {type(copy_source)}.' ) def _extra_upload_part_args(self, extra_args): diff --git a/s3transfer/futures.py b/s3transfer/futures.py index 540f5cc6..68775d04 100644 --- a/s3transfer/futures.py +++ b/s3transfer/futures.py @@ -293,8 +293,8 @@ def _transition_to_non_done_state(self, desired_state): with self._lock: if self.done(): raise RuntimeError( - 'Unable to transition from done state %s to non-done ' - 'state %s.' % (self.status, desired_state) + f'Unable to transition from done state {self.status} to non-done ' + f'state {desired_state}.' ) self._status = desired_state @@ -396,7 +396,7 @@ def _run_callback(self, callback): # We do not want a callback interrupting the process, especially # in the failure cleanups. So log and catch, the exception. except Exception: - logger.debug("Exception raised in %s." % callback, exc_info=True) + logger.debug(f"Exception raised in {callback}.", exc_info=True) class BoundedExecutor: diff --git a/s3transfer/manager.py b/s3transfer/manager.py index ab9a210f..8db9a411 100644 --- a/s3transfer/manager.py +++ b/s3transfer/manager.py @@ -149,8 +149,8 @@ def _validate_attrs_are_nonzero(self): for attr, attr_val in self.__dict__.items(): if attr_val is not None and attr_val <= 0: raise ValueError( - 'Provided parameter %s of value %s must be greater than ' - '0.' % (attr, attr_val) + f'Provided parameter {attr} of value {attr_val} must ' + 'be greater than 0.' ) @@ -492,16 +492,16 @@ def _validate_if_bucket_supported(self, bucket): match = pattern.match(bucket) if match: raise ValueError( - 'TransferManager methods do not support %s ' - 'resource. Use direct client calls instead.' % resource + f'TransferManager methods do not support {resource} ' + 'resource. Use direct client calls instead.' ) def _validate_all_known_args(self, actual, allowed): for kwarg in actual: if kwarg not in allowed: raise ValueError( - "Invalid extra_args key '%s', " - "must be one of: %s" % (kwarg, ', '.join(allowed)) + "Invalid extra_args key '{}', " + "must be one of: {}".format(kwarg, ', '.join(allowed)) ) def _add_operation_defaults(self, bucket, extra_args): diff --git a/s3transfer/subscribers.py b/s3transfer/subscribers.py index eeed5e43..fe773233 100644 --- a/s3transfer/subscribers.py +++ b/s3transfer/subscribers.py @@ -36,14 +36,13 @@ def _validate_subscriber_methods(cls): subscriber_method = getattr(cls, 'on_' + subscriber_type) if not callable(subscriber_method): raise InvalidSubscriberMethodError( - 'Subscriber method %s must be callable.' - % subscriber_method + f'Subscriber method {subscriber_method} must be callable.' ) if not accepts_kwargs(subscriber_method): raise InvalidSubscriberMethodError( - 'Subscriber method %s must accept keyword ' - 'arguments (**kwargs)' % subscriber_method + f'Subscriber method {subscriber_method} must accept keyword ' + 'arguments (**kwargs)' ) def on_queued(self, future, **kwargs): diff --git a/s3transfer/tasks.py b/s3transfer/tasks.py index 4917dfc9..4183715a 100644 --- a/s3transfer/tasks.py +++ b/s3transfer/tasks.py @@ -96,10 +96,7 @@ def __repr__(self): main_kwargs_to_display = self._get_kwargs_with_params_to_include( self._main_kwargs, params_to_display ) - return ( - f'{self.__class__.__name__}(transfer_id=' - f'{self._transfer_coordinator.transfer_id}, {main_kwargs_to_display})' - ) + return f'{self.__class__.__name__}(transfer_id={self._transfer_coordinator.transfer_id}, {main_kwargs_to_display})' @property def transfer_id(self): diff --git a/s3transfer/utils.py b/s3transfer/utils.py index 62ecf39c..98742236 100644 --- a/s3transfer/utils.py +++ b/s3transfer/utils.py @@ -634,7 +634,7 @@ def acquire(self, tag, blocking=True): """ logger.debug("Acquiring %s", tag) if not self._semaphore.acquire(blocking): - raise NoResourcesAvailable("Cannot acquire tag '%s'" % tag) + raise NoResourcesAvailable(f"Cannot acquire tag '{tag}'") def release(self, tag, acquire_token): """Release the semaphore @@ -692,7 +692,7 @@ def acquire(self, tag, blocking=True): try: if self._count == 0: if not blocking: - raise NoResourcesAvailable("Cannot acquire tag '%s'" % tag) + raise NoResourcesAvailable(f"Cannot acquire tag '{tag}'") else: while self._count == 0: self._condition.wait() @@ -714,7 +714,7 @@ def release(self, tag, acquire_token): self._condition.acquire() try: if tag not in self._tag_sequences: - raise ValueError("Attempted to release unknown tag: %s" % tag) + raise ValueError(f"Attempted to release unknown tag: {tag}") max_sequence = self._tag_sequences[tag] if self._lowest_sequence[tag] == sequence_number: # We can immediately process this request and free up @@ -741,7 +741,7 @@ def release(self, tag, acquire_token): else: raise ValueError( "Attempted to release unknown sequence number " - "%s for tag: %s" % (sequence_number, tag) + f"{sequence_number} for tag: {tag}" ) finally: self._condition.release() @@ -779,13 +779,13 @@ def _adjust_for_chunksize_limits(self, current_chunksize): if current_chunksize > self.max_size: logger.debug( "Chunksize greater than maximum chunksize. " - "Setting to %s from %s." % (self.max_size, current_chunksize) + f"Setting to {self.max_size} from {current_chunksize}." ) return self.max_size elif current_chunksize < self.min_size: logger.debug( "Chunksize less than minimum chunksize. " - "Setting to %s from %s." % (self.min_size, current_chunksize) + f"Setting to {self.min_size} from {current_chunksize}." ) return self.min_size else: @@ -802,8 +802,7 @@ def _adjust_for_max_parts(self, current_chunksize, file_size): if chunksize != current_chunksize: logger.debug( "Chunksize would result in the number of parts exceeding the " - "maximum. Setting to %s from %s." - % (chunksize, current_chunksize) + f"maximum. Setting to {chunksize} from {current_chunksize}." ) return chunksize diff --git a/scripts/ci/install b/scripts/ci/install index 2f3a8f97..8b4e8b0f 100755 --- a/scripts/ci/install +++ b/scripts/ci/install @@ -44,4 +44,4 @@ if __name__ == "__main__": package = os.path.join('dist', wheel_dist) if args.extras: package = f"'{package}[{args.extras}]'" - run('pip install %s' % package) + run(f'pip install {package}') diff --git a/scripts/performance/benchmark b/scripts/performance/benchmark index af8dbb21..d1aa8d7a 100755 --- a/scripts/performance/benchmark +++ b/scripts/performance/benchmark @@ -35,7 +35,7 @@ elif sys.platform == 'darwin': else: # TODO: Add support for windows. This would require figuring out what # interface to use on windows. - raise RuntimeError('Script cannot be run on %s' % sys.platform) + raise RuntimeError(f'Script cannot be run on {sys.platform}') def benchmark(args): diff --git a/scripts/performance/benchmark-download b/scripts/performance/benchmark-download index 30fdba1c..14bab5be 100755 --- a/scripts/performance/benchmark-download +++ b/scripts/performance/benchmark-download @@ -63,7 +63,7 @@ def human_readable_to_bytes(value): try: return int(value) except ValueError: - raise ValueError("Invalid size value: %s" % value) + raise ValueError(f"Invalid size value: {value}") else: multiplier = SIZE_SUFFIX[suffix] return int(value[: -len(suffix)]) * multiplier @@ -97,8 +97,8 @@ def benchmark_download(args): upload_file(client, temp_file, args.s3_bucket) download_file_script = ( - './download-file --file-name %s --file-type %s --s3-bucket %s ' - '--s3-key %s' % (temp_file, args.file_type, args.s3_bucket, s3_key) + f'./download-file --file-name {temp_file} --file-type {args.file_type} --s3-bucket {args.s3_bucket} ' + f'--s3-key {s3_key}' ) benchmark_args = ['./benchmark', download_file_script] if args.output_file: diff --git a/scripts/performance/benchmark-upload b/scripts/performance/benchmark-upload index 64680535..bc67e1f0 100755 --- a/scripts/performance/benchmark-upload +++ b/scripts/performance/benchmark-upload @@ -61,7 +61,7 @@ def human_readable_to_bytes(value): try: return int(value) except ValueError: - raise ValueError("Invalid size value: %s" % value) + raise ValueError(f"Invalid size value: {value}") else: multiplier = SIZE_SUFFIX[suffix] return int(value[: -len(suffix)]) * multiplier @@ -87,9 +87,8 @@ def benchmark_upload(args): create_file(source_file, args.file_size) upload_file_script = ( - './upload-file --file-name %s --file-type %s --s3-bucket %s ' - '--s3-key %s' - % (source_file, args.file_type, args.s3_bucket, TEMP_KEY) + f'./upload-file --file-name {source_file} --file-type {args.file_type} --s3-bucket {args.s3_bucket} ' + f'--s3-key {TEMP_KEY}' ) benchmark_args = ['./benchmark', upload_file_script] if args.output_file: diff --git a/scripts/performance/summarize b/scripts/performance/summarize index fb3a5cc9..dd3abc33 100755 --- a/scripts/performance/summarize +++ b/scripts/performance/summarize @@ -168,13 +168,13 @@ class Summarizer: table = [ [ 'Total Time (seconds)', - '%.3f' % self.total_time, + f'{self.total_time:.3f}', self.std_dev_total_time, ], ['Maximum Memory', h(self.max_memory), h(self.std_dev_max_memory)], [ 'Maximum CPU (percent)', - '%.1f' % self.max_cpu, + f'{self.max_cpu:.1f}', self.std_dev_max_cpu, ], [ @@ -184,14 +184,14 @@ class Summarizer: ], [ 'Average CPU (percent)', - '%.1f' % self.average_cpu, + f'{self.average_cpu:.1f}', self.std_dev_average_cpu, ], ] return tabulate( table, headers=[ - 'Metric over %s run(s)' % (self.total_files), + f'Metric over {self.total_files} run(s)', 'Mean', 'Standard Deviation', ], @@ -238,8 +238,8 @@ class Summarizer: def _validate_row(self, row, filename): if not row: raise RuntimeError( - 'Row: %s could not be processed. The CSV file (%s) may be ' - 'empty.' % (row, filename) + f'Row: {row} could not be processed. The CSV file ({filename}) may be ' + 'empty.' ) def process_data_row(self, row): diff --git a/scripts/stress/timeout b/scripts/stress/timeout index fa5bba52..71001824 100755 --- a/scripts/stress/timeout +++ b/scripts/stress/timeout @@ -25,7 +25,7 @@ import psutil class TimeoutException(Exception): def __init__(self, timeout_len): - msg = 'Script failed to complete within %s seconds' % timeout_len + msg = f'Script failed to complete within {timeout_len} seconds' Exception.__init__(self, msg) diff --git a/tests/functional/test_copy.py b/tests/functional/test_copy.py index d13755d8..4120377a 100644 --- a/tests/functional/test_copy.py +++ b/tests/functional/test_copy.py @@ -396,7 +396,7 @@ def add_upload_part_copy_responses_with_default_expected_params( if extra_expected_params: if 'ChecksumAlgorithm' in extra_expected_params: name = extra_expected_params['ChecksumAlgorithm'] - checksum_member = 'Checksum%s' % name.upper() + checksum_member = f'Checksum{name.upper()}' response = upload_part_response['service_response'] response['CopyPartResult'][checksum_member] = 'sum%s==' % ( i + 1 diff --git a/tests/functional/test_crt.py b/tests/functional/test_crt.py index 352e5854..ff1b413f 100644 --- a/tests/functional/test_crt.py +++ b/tests/functional/test_crt.py @@ -77,7 +77,7 @@ def setUp(self): 'myfile', self.expected_content, mode='wb' ) self.expected_path = "/" + self.bucket + "/" + self.key - self.expected_host = "s3.%s.amazonaws.com" % (self.region) + self.expected_host = f"s3.{self.region}.amazonaws.com" self.expected_s3express_host = f'{self.s3express_bucket}.s3express-usw2-az5.us-west-2.amazonaws.com' self.expected_s3express_path = f'/{self.key}' self.s3_request = mock.Mock(awscrt.s3.S3Request) diff --git a/tests/functional/test_download.py b/tests/functional/test_download.py index a36a2356..ccbac406 100644 --- a/tests/functional/test_download.py +++ b/tests/functional/test_download.py @@ -141,7 +141,7 @@ def test_download_temporary_file_does_not_exist(self): # Make sure the file exists self.assertTrue(os.path.exists(self.filename)) # Make sure the random temporary file does not exist - possible_matches = glob.glob('%s*' % self.filename + os.extsep) + possible_matches = glob.glob(f'{self.filename}*' + os.extsep) self.assertEqual(possible_matches, []) def test_download_for_fileobj(self): @@ -201,7 +201,7 @@ def test_download_cleanup_on_failure(self): future.result() # Make sure the actual file and the temporary do not exist # by globbing for the file and any of its extensions - possible_matches = glob.glob('%s*' % self.filename) + possible_matches = glob.glob(f'{self.filename}*') self.assertEqual(possible_matches, []) def test_download_with_nonexistent_directory(self): diff --git a/tests/functional/test_processpool.py b/tests/functional/test_processpool.py index 7b45d1f3..baf0232d 100644 --- a/tests/functional/test_processpool.py +++ b/tests/functional/test_processpool.py @@ -212,7 +212,7 @@ def test_cleans_up_tempfile_on_failure(self): ) self.assertFalse(os.path.exists(self.filename)) # Any tempfile should have been erased as well - possible_matches = glob.glob('%s*' % self.filename + os.extsep) + possible_matches = glob.glob(f'{self.filename}*' + os.extsep) self.assertEqual(possible_matches, []) def test_validates_extra_args(self): diff --git a/tests/functional/test_upload.py b/tests/functional/test_upload.py index 021b409b..e3b54899 100644 --- a/tests/functional/test_upload.py +++ b/tests/functional/test_upload.py @@ -92,7 +92,7 @@ def collect_body(self, params, model, **kwargs): data=params['Body'], ) self.client.meta.events.emit( - 'request-created.s3.%s' % model.name, + f'request-created.s3.{model.name}', request=request, operation_name=model.name, ) @@ -398,7 +398,7 @@ def add_upload_part_responses_with_default_expected_params( # If ChecksumAlgorithm is present stub the response checksums if 'ChecksumAlgorithm' in extra_expected_params: name = extra_expected_params['ChecksumAlgorithm'] - checksum_member = 'Checksum%s' % name.upper() + checksum_member = f'Checksum{name.upper()}' response = upload_part_response['service_response'] response[checksum_member] = 'sum%s==' % (i + 1) diff --git a/tests/integration/test_crt.py b/tests/integration/test_crt.py index 7f16d85e..a13ff17b 100644 --- a/tests/integration/test_crt.py +++ b/tests/integration/test_crt.py @@ -508,6 +508,6 @@ def test_download_cancel(self): future.result() self.assertEqual(err.name, 'AWS_ERROR_S3_CANCELED') - possible_matches = glob.glob('%s*' % download_path) + possible_matches = glob.glob(f'{download_path}*') self.assertEqual(possible_matches, []) self._assert_subscribers_called() diff --git a/tests/integration/test_download.py b/tests/integration/test_download.py index 9561f61d..da08514e 100644 --- a/tests/integration/test_download.py +++ b/tests/integration/test_download.py @@ -98,7 +98,7 @@ def test_large_download_exits_quicky_on_exception(self): future.cancel() raise RuntimeError( "Download transfer did not start after waiting for " - "%s seconds." % timeout + f"{timeout} seconds." ) # Raise an exception which should cause the preceding # download to cancel and exit quickly @@ -124,7 +124,7 @@ def test_large_download_exits_quicky_on_exception(self): # Make sure the actual file and the temporary do not exist # by globbing for the file and any of its extensions - possible_matches = glob.glob('%s*' % download_path) + possible_matches = glob.glob(f'{download_path}*') self.assertEqual(possible_matches, []) @skip_if_using_serial_implementation( @@ -182,7 +182,7 @@ def test_many_files_exits_quicky_on_exception(self): # For the transfer that did get cancelled, make sure the temporary # file got removed. - possible_matches = glob.glob('%s*' % future.meta.call_args.fileobj) + possible_matches = glob.glob(f'{future.meta.call_args.fileobj}*') self.assertEqual(possible_matches, []) def test_progress_subscribers_on_download(self): @@ -280,5 +280,5 @@ def test_download_to_special_file(self): except Exception as e: self.fail( 'Should have been able to download to /dev/null but received ' - 'following exception %s' % e + f'following exception {e}' ) diff --git a/tests/integration/test_processpool.py b/tests/integration/test_processpool.py index 4287f3b1..86bbc62f 100644 --- a/tests/integration/test_processpool.py +++ b/tests/integration/test_processpool.py @@ -101,7 +101,7 @@ def test_large_download_exits_quickly_on_exception(self): # Make sure the actual file and the temporary do not exist # by globbing for the file and any of its extensions - possible_matches = glob.glob('%s*' % download_path) + possible_matches = glob.glob(f'{download_path}*') self.assertEqual(possible_matches, []) def test_many_files_exits_quickly_on_exception(self): @@ -141,5 +141,5 @@ def test_many_files_exits_quickly_on_exception(self): # For the transfer that did get cancelled, make sure the temporary # file got removed. - possible_matches = glob.glob('%s*' % base_filename) + possible_matches = glob.glob(f'{base_filename}*') self.assertEqual(possible_matches, []) diff --git a/tests/integration/test_upload.py b/tests/integration/test_upload.py index cf6a22f8..a12d1818 100644 --- a/tests/integration/test_upload.py +++ b/tests/integration/test_upload.py @@ -84,7 +84,7 @@ def test_large_upload_exits_quicky_on_exception(self): future.cancel() raise RuntimeError( "Download transfer did not start after waiting for " - "%s seconds." % timeout + f"{timeout} seconds." ) # Raise an exception which should cause the preceding # download to cancel and exit quickly diff --git a/tests/unit/test_crt.py b/tests/unit/test_crt.py index dc5b745c..477c91df 100644 --- a/tests/unit/test_crt.py +++ b/tests/unit/test_crt.py @@ -103,7 +103,7 @@ def setUp(self): self.files = FileCreator() self.filename = self.files.create_file('myfile', 'my content') self.expected_path = "/" + self.bucket + "/" + self.key - self.expected_host = "s3.%s.amazonaws.com" % (self.region) + self.expected_host = f"s3.{self.region}.amazonaws.com" def tearDown(self): self.files.remove_all() diff --git a/tests/unit/test_manager.py b/tests/unit/test_manager.py index b377b200..78c3b8d7 100644 --- a/tests/unit/test_manager.py +++ b/tests/unit/test_manager.py @@ -132,7 +132,7 @@ def test_wait_does_not_propogate_exceptions_from_result(self): try: self.coordinator_controller.wait() except FutureResultException as e: - self.fail('%s should not have been raised.' % e) + self.fail(f'{e} should not have been raised.') def test_wait_can_be_interrupted(self): inject_interrupt_coordinator = TransferCoordinatorWithInterrupt() diff --git a/tests/unit/test_subscribers.py b/tests/unit/test_subscribers.py index a3d5a435..e66461ab 100644 --- a/tests/unit/test_subscribers.py +++ b/tests/unit/test_subscribers.py @@ -54,7 +54,7 @@ def test_can_call_base_subscriber_method(self): except Exception as e: self.fail( 'Should be able to call base class subscriber method. ' - 'instead got: %s' % e + f'instead got: {e}' ) def test_subclass_can_have_and_call_additional_methods(self): diff --git a/tests/unit/test_upload.py b/tests/unit/test_upload.py index 4c523011..84b1e67d 100644 --- a/tests/unit/test_upload.py +++ b/tests/unit/test_upload.py @@ -50,7 +50,7 @@ class InterruptionError(Exception): class OSUtilsExceptionOnFileSize(OSUtils): def get_file_size(self, filename): raise AssertionError( - "The file %s should not have been stated" % filename + f"The file {filename} should not have been stated" ) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 13a079d9..59996696 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -282,7 +282,7 @@ def test_remove_file_ignores_errors(self): try: OSUtils().remove_file(non_existent_file) except OSError as e: - self.fail('OSError should have been caught: %s' % e) + self.fail(f'OSError should have been caught: {e}') def test_remove_file_proxies_remove_file(self): OSUtils().remove_file(self.filename) @@ -306,7 +306,7 @@ def test_get_temp_filename(self): filename = 'myfile' self.assertIsNotNone( re.match( - r'%s\.[0-9A-Fa-f]{8}$' % filename, + rf'{filename}\.[0-9A-Fa-f]{{8}}$', OSUtils().get_temp_filename(filename), ) ) From 5879a1f6c0d61731f62ddc05a8cb5704c047b498 Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Wed, 19 Jun 2024 14:30:29 -0600 Subject: [PATCH 4/9] Add I for isort functionality --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c93b2674..35a5af32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ target-version = "py38" # Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. # Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or # McCabe complexity (`C901`) by default. -select = ["E4", "E7", "E9", "F", "UP"] +select = ["E4", "E7", "E9", "F", "I", "UP"] ignore = [] # Allow fix for all enabled rules (when `--fix`) is provided. From a414253c811a1ecfdba901e142aa06381bf7b96a Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Fri, 21 Jun 2024 14:19:35 -0600 Subject: [PATCH 5/9] Updating contributing guide --- CONTRIBUTING.rst | 18 ++++++++---------- setup.cfg | 3 --- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 9dc027cd..34c911c1 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -35,10 +35,11 @@ ideal report includes: Codestyle --------- -This project uses flake8 to enforce codstyle requirements. We've codified this -process using a tool called `pre-commit `__. pre-commit -allows us to specify a config file with all tools required for code linting, -and surfaces either a git commit hook, or single command, for enforcing these. +This project uses `ruff `__ to enforce +codstyle requirements. We've codified this process using a tool called +`pre-commit `__. pre-commit allows us to specify a +config file with all tools required for code linting, and surfaces either a +git commit hook, or single command, for enforcing these. To validate your PR prior to publishing, you can use the following `installation guide `__ to setup pre-commit. @@ -51,10 +52,7 @@ to automatically perform the codestyle validation: $ pre-commit run This will automatically perform simple updates (such as white space clean up) -and provide a list of any failing flake8 checks. After these are addressed, +and provide a list of any failing checks. After these are addressed, you can commit the changes prior to publishing the PR. -These checks are also included in our CI setup under the "Lint" workflow which will provide output on Github for anything missed locally. - -See the `flake8` section of the -`setup.cfg `__ for the -currently enforced rules. +These checks are also included in our CI setup under the "Lint" workflow which +will provide output on Github for anything missed locally. diff --git a/setup.cfg b/setup.cfg index 3e3c0567..b997161e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,6 +7,3 @@ requires_dist = [options.extras_require] crt = botocore[crt]>=1.33.2,<2.0a0 - -[flake8] -ignore = E203,E226,E501,W503,W504 From c87f32193809e6b7f178b8cf2747c91935dc8308 Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Mon, 22 Jul 2024 15:02:27 -0700 Subject: [PATCH 6/9] Temporary fix for setuptools 70.x (#311) --- scripts/ci/install-dev-deps | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/install-dev-deps b/scripts/ci/install-dev-deps index 74773764..903b0556 100755 --- a/scripts/ci/install-dev-deps +++ b/scripts/ci/install-dev-deps @@ -27,5 +27,5 @@ def run(command): if __name__ == "__main__": with cd(REPO_ROOT): if sys.version_info[:2] >= (3, 12): - run("pip install setuptools") + run("pip install setuptools==67.8.0") run("pip install -r requirements-dev-lock.txt") From da68b50bb5a6b0c342ad0d87f9b1f80ab81dffce Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Mon, 29 Jul 2024 05:46:23 -0700 Subject: [PATCH 7/9] Update setuptools (#313) --- requirements-dev-lock.txt | 34 +++++++++++++++------------------- requirements-dev.txt | 3 ++- scripts/ci/install-dev-deps | 8 +++++++- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/requirements-dev-lock.txt b/requirements-dev-lock.txt index d0972326..8b5a0229 100644 --- a/requirements-dev-lock.txt +++ b/requirements-dev-lock.txt @@ -7,9 +7,9 @@ atomicwrites==1.4.1 \ --hash=sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11 # via -r requirements-dev.txt -colorama==0.4.5 \ - --hash=sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da \ - --hash=sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4 +colorama==0.4.6 \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 # via -r requirements-dev.txt coverage[toml]==7.2.7 \ --hash=sha256:06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f \ @@ -75,21 +75,21 @@ coverage[toml]==7.2.7 \ # via # -r requirements-dev.txt # pytest-cov -exceptiongroup==1.1.3 \ - --hash=sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9 \ - --hash=sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3 +exceptiongroup==1.2.2 \ + --hash=sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b \ + --hash=sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc # via pytest -iniconfig==1.1.1 \ - --hash=sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3 \ - --hash=sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32 +iniconfig==2.0.0 \ + --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ + --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 # via pytest -packaging==21.3 \ - --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ - --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 +packaging==24.1 \ + --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ + --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 # via pytest -pluggy==1.4.0 \ - --hash=sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981 \ - --hash=sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be +pluggy==1.5.0 \ + --hash=sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1 \ + --hash=sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669 # via pytest psutil==4.4.2 \ --hash=sha256:10fbb631142a3200623f4ab49f8bf82c32b79b8fe179f6056d01da3dfc589da1 \ @@ -102,10 +102,6 @@ psutil==4.4.2 \ --hash=sha256:e423dd9cb12256c742d1d56ec38bc7d2a7fa09287c82c41e475e68b9f932c2af \ --hash=sha256:e44d6b758a96539e3e02336430d3f85263d43c470c5bad93572e9b6a86c67f76 # via -r requirements-dev.txt -pyparsing==3.0.9 \ - --hash=sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb \ - --hash=sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc - # via packaging pytest==8.1.1 \ --hash=sha256:2a8386cfc11fa9d2c50ee7b2a57e7d898ef90470a7a34c4b949ff59662bb78b7 \ --hash=sha256:ac978141a75948948817d360297b7aae0fcb9d6ff6bc9ec6d514b85d5a65c044 diff --git a/requirements-dev.txt b/requirements-dev.txt index eb837d8e..0fba7747 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,8 @@ psutil>=4.1.0,<5.0.0 tabulate==0.7.5 coverage==7.2.7 wheel==0.43.0 -setuptools==67.8.0;python_version>="3.12" +setuptools==71.1.0;python_version>="3.12" +packaging==24.1;python_version>="3.12" # Requirement for setuptools>=71 # Pytest specific deps pytest==8.1.1 diff --git a/scripts/ci/install-dev-deps b/scripts/ci/install-dev-deps index 903b0556..71e9badc 100755 --- a/scripts/ci/install-dev-deps +++ b/scripts/ci/install-dev-deps @@ -27,5 +27,11 @@ def run(command): if __name__ == "__main__": with cd(REPO_ROOT): if sys.version_info[:2] >= (3, 12): - run("pip install setuptools==67.8.0") + # Python 3.12+ no longer includes setuptools by default. + + # Setuptools 71+ now prefers already installed versions + # of packaging _and_ broke the API for packaging<22.0. + # We'll pin to match what's in requirements-dev.txt. + run("pip install setuptools==71.1.0 packaging==24.1") + run("pip install -r requirements-dev-lock.txt") From 0d7a87f8c103488dce93c2dfece51c802cf2a0cc Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Mon, 7 Oct 2024 14:37:28 -0700 Subject: [PATCH 8/9] Add support for Python 3.13 (#315) --- .changes/next-release/enhancement-Python-9497.json | 5 +++++ .github/workflows/run-crt-test.yml | 4 +++- .github/workflows/run-tests.yml | 4 +++- setup.py | 1 + tox.ini | 2 +- 5 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 .changes/next-release/enhancement-Python-9497.json diff --git a/.changes/next-release/enhancement-Python-9497.json b/.changes/next-release/enhancement-Python-9497.json new file mode 100644 index 00000000..e5045f02 --- /dev/null +++ b/.changes/next-release/enhancement-Python-9497.json @@ -0,0 +1,5 @@ +{ + "type": "enhancement", + "category": "Python", + "description": "Added provisional Python 3.13 support to s3transfer" +} diff --git a/.github/workflows/run-crt-test.yml b/.github/workflows/run-crt-test.yml index 50141892..d171a04f 100644 --- a/.github/workflows/run-crt-test.yml +++ b/.github/workflows/run-crt-test.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13-dev"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] os: [ubuntu-latest, macOS-latest, windows-latest] # Python 3.8 and 3.9 do not run on m1 hardware which is now standard for # macOS-latest. @@ -30,6 +30,8 @@ jobs: uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 with: python-version: ${{ matrix.python-version }} + cache: 'pip' + allow-prereleases: true - name: Install dependencies and CRT run: | python scripts/ci/install --extras crt diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 92a8f7de..267ef776 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13-dev"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] os: [ubuntu-latest, macOS-latest, windows-latest] # Python 3.8 and 3.9 do not run on m1 hardware which is now standard for # macOS-latest. @@ -33,6 +33,8 @@ jobs: uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 with: python-version: ${{ matrix.python-version }} + cache: 'pip' + allow-prereleases: true - name: Install dependencies run: | python scripts/ci/install diff --git a/setup.py b/setup.py index 179a405c..b881c852 100644 --- a/setup.py +++ b/setup.py @@ -47,5 +47,6 @@ def get_version(): 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', ], ) diff --git a/tox.ini b/tox.ini index 780cff37..91ef2cf0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38,py39,py310,py311,py312 +envlist = py38,py39,py310,py311,py312,py313 # Comment to build sdist and install into virtualenv # This is helpful to test installation but takes extra time From bf87770cce56770638e565f8d6929039b6a7e444 Mon Sep 17 00:00:00 2001 From: aws-sdk-python-automation Date: Tue, 8 Oct 2024 18:06:31 +0000 Subject: [PATCH 9/9] Bumping version to 0.10.3 --- .changes/0.10.3.json | 7 +++++++ .changes/next-release/enhancement-Python-9497.json | 5 ----- CHANGELOG.rst | 6 ++++++ s3transfer/__init__.py | 2 +- 4 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 .changes/0.10.3.json delete mode 100644 .changes/next-release/enhancement-Python-9497.json diff --git a/.changes/0.10.3.json b/.changes/0.10.3.json new file mode 100644 index 00000000..b9993009 --- /dev/null +++ b/.changes/0.10.3.json @@ -0,0 +1,7 @@ +[ + { + "category": "Python", + "description": "Added provisional Python 3.13 support to s3transfer", + "type": "enhancement" + } +] \ No newline at end of file diff --git a/.changes/next-release/enhancement-Python-9497.json b/.changes/next-release/enhancement-Python-9497.json deleted file mode 100644 index e5045f02..00000000 --- a/.changes/next-release/enhancement-Python-9497.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "enhancement", - "category": "Python", - "description": "Added provisional Python 3.13 support to s3transfer" -} diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c8f12ef3..f5ea99b3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,12 @@ CHANGELOG ========= +0.10.3 +====== + +* enhancement:Python: Added provisional Python 3.13 support to s3transfer + + 0.10.2 ====== diff --git a/s3transfer/__init__.py b/s3transfer/__init__.py index d41de360..056492cb 100644 --- a/s3transfer/__init__.py +++ b/s3transfer/__init__.py @@ -145,7 +145,7 @@ def __call__(self, bytes_amount): from s3transfer.exceptions import RetriesExceededError, S3UploadFailedError __author__ = 'Amazon Web Services' -__version__ = '0.10.2' +__version__ = '0.10.3' class NullHandler(logging.Handler):