From 169d673a0e8b31fb13e7bf0bdd94c017bd508b0b Mon Sep 17 00:00:00 2001
From: Daniel Gafni <danielgafni16@gmail.com>
Date: Mon, 14 Oct 2024 14:16:15 +0200
Subject: [PATCH] [dagster-aws, dagster-pipes] Add PipesS3LogReader (#25224)

## Summary & Motivation

Add `PipesS3LogReader` which can read logs from known S3 objects.

## How I Tested These Changes

Added a test
---
 pyright/alt-1/requirements-pinned.txt         | 52 +++++-----
 pyright/master/requirements-pinned.txt        | 97 +++++++++++--------
 .../dagster-aws/dagster_aws/pipes/__init__.py |  2 +
 .../dagster_aws/pipes/message_readers.py      | 80 ++++++++++++++-
 .../pipes_tests/test_pipes.py                 | 31 ++++++
 python_modules/libraries/dagster-aws/setup.py |  2 +-
 6 files changed, 197 insertions(+), 67 deletions(-)

diff --git a/pyright/alt-1/requirements-pinned.txt b/pyright/alt-1/requirements-pinned.txt
index 3b5c16ac01e4d..9adabce9b43d1 100644
--- a/pyright/alt-1/requirements-pinned.txt
+++ b/pyright/alt-1/requirements-pinned.txt
@@ -1,21 +1,20 @@
 agate==1.9.1
 aiobotocore==2.15.1
-aiofile==3.8.8
+aiofile==3.9.0
 aiohappyeyeballs==2.4.3
-aiohttp==3.10.8
+aiohttp==3.10.9
 aioitertools==0.12.0
 aiosignal==1.3.1
 alembic==1.13.3
 aniso8601==9.0.1
 annotated-types==0.7.0
 anyio==4.6.0
-appnope==0.1.4
 argon2-cffi==23.1.0
 argon2-cffi-bindings==21.2.0
 arrow==1.3.0
 asn1crypto==1.5.1
 -e examples/assets_pandas_type_metadata
-astroid==3.3.4
+astroid==3.3.5
 asttokens==2.4.1
 async-lru==2.0.4
 attrs==24.2.0
@@ -25,16 +24,16 @@ backports-tarfile==1.2.0
 beautifulsoup4==4.12.3
 bleach==6.1.0
 boto3==1.35.23
-boto3-stubs-lite==1.35.31
+boto3-stubs-lite==1.35.36
 botocore==1.35.23
-botocore-stubs==1.35.31
+botocore-stubs==1.35.36
 buildkite-test-collector==0.1.9
 cachetools==5.5.0
 caio==0.9.17
 certifi==2024.8.30
 cffi==1.17.1
 chardet==5.2.0
-charset-normalizer==3.3.2
+charset-normalizer==3.4.0
 click==8.1.7
 colorama==0.4.6
 coloredlogs==14.0
@@ -69,7 +68,7 @@ db-dtypes==1.3.0
 dbt-adapters==1.3.2
 dbt-common==1.3.0
 dbt-core==1.8.7
-dbt-duckdb==1.8.4
+dbt-duckdb==1.9.0
 dbt-extractor==0.5.1
 dbt-semantic-interfaces==0.5.1
 dbt-snowflake==1.8.3
@@ -78,7 +77,7 @@ decopatch==1.4.10
 decorator==5.1.1
 deepdiff==8.0.1
 defusedxml==0.7.1
-deltalake==0.20.1
+deltalake==0.20.2
 dill==0.3.9
 distlib==0.3.8
 docker==7.1.0
@@ -94,7 +93,7 @@ frozenlist==1.4.1
 fsspec==2024.3.0
 gcsfs==0.8.0
 google-api-core==2.20.0
-google-api-python-client==2.147.0
+google-api-python-client==2.148.0
 google-auth==2.35.0
 google-auth-httplib2==0.2.0
 google-auth-oauthlib==1.2.1
@@ -108,6 +107,7 @@ gql==3.5.0
 graphene==3.3
 graphql-core==3.2.4
 graphql-relay==3.2.0
+greenlet==3.1.1
 grpcio==1.66.2
 grpcio-health-checking==1.62.3
 grpcio-status==1.62.3
@@ -118,7 +118,7 @@ httplib2==0.22.0
 httptools==0.6.1
 httpx==0.27.2
 humanfriendly==10.0
-hypothesis==6.112.2
+hypothesis==6.113.0
 idna==3.10
 importlib-metadata==6.11.0
 iniconfig==2.0.0
@@ -131,13 +131,14 @@ jaraco-classes==3.4.0
 jaraco-context==6.0.1
 jaraco-functools==4.1.0
 jedi==0.19.1
+jeepney==0.8.0
 jinja2==3.1.4
 jmespath==1.0.1
 joblib==1.4.2
 json5==0.9.25
 jsonpointer==3.0.0
 jsonschema==4.23.0
-jsonschema-specifications==2023.12.1
+jsonschema-specifications==2024.10.1
 jupyter-client==8.6.3
 jupyter-core==5.7.2
 jupyter-events==0.10.0
@@ -154,7 +155,7 @@ logbook==1.5.3
 makefun==1.15.6
 mako==1.3.5
 markdown-it-py==3.0.0
-markupsafe==2.1.5
+markupsafe==3.0.1
 mashumaro==3.13.1
 matplotlib==3.9.2
 matplotlib-inline==0.1.7
@@ -172,6 +173,7 @@ mypy==1.11.2
 mypy-boto3-ecs==1.35.21
 mypy-boto3-emr-serverless==1.35.25
 mypy-boto3-glue==1.35.25
+mypy-boto3-s3==1.35.32
 mypy-extensions==1.0.0
 mypy-protobuf==3.6.0
 nbclient==0.10.0
@@ -181,7 +183,7 @@ nest-asyncio==1.6.0
 networkx==3.3
 nodeenv==1.9.1
 notebook-shim==0.2.4
-numpy==2.1.1
+numpy==2.1.2
 oauth2client==4.1.3
 oauthlib==3.2.2
 objgraph==3.6.1
@@ -205,6 +207,7 @@ polars==1.9.0
 -e examples/project_fully_featured
 prometheus-client==0.21.0
 prompt-toolkit==3.0.48
+propcache==0.2.0
 proto-plus==1.24.0
 protobuf==4.25.5
 psutil==6.0.0
@@ -249,14 +252,15 @@ requests-toolbelt==1.0.0
 responses==0.23.1
 rfc3339-validator==0.1.4
 rfc3986-validator==0.1.1
-rich==13.9.1
+rich==13.9.2
 rpds-py==0.20.0
 rsa==4.9
 s3fs==2024.3.0
-s3transfer==0.10.2
+s3transfer==0.10.3
 scikit-learn==1.5.2
 scipy==1.14.1
 seaborn==0.13.2
+secretstorage==3.3.3
 send2trash==1.8.3
 setuptools==75.1.0
 shellingham==1.5.4
@@ -268,13 +272,13 @@ snowflake-sqlalchemy==1.5.1
 sortedcontainers==2.4.0
 soupsieve==2.6
 sqlalchemy==1.4.54
-sqlglot==25.24.1
+sqlglot==25.24.5
 sqlglotrs==0.2.12
 sqlparse==0.5.1
 stack-data==0.6.3
 starlette==0.39.2
 structlog==24.4.0
-syrupy==4.7.1
+syrupy==4.7.2
 tabulate==0.9.0
 terminado==0.18.1
 text-unidecode==1.3
@@ -284,7 +288,7 @@ tomli==2.0.2
 tomlkit==0.13.2
 toposort==1.10
 tornado==6.4.1
-tox==4.21.0
+tox==4.21.2
 tqdm==4.66.5
 traitlets==5.14.3
 typeguard==4.3.0
@@ -300,14 +304,14 @@ types-mock==5.1.0.20240425
 types-paramiko==3.5.0.20240928
 types-protobuf==5.28.0.20240924
 types-pyopenssl==24.1.0.20240722
-types-python-dateutil==2.9.0.20240906
-types-pytz==2024.2.0.20240913
+types-python-dateutil==2.9.0.20241003
+types-pytz==2024.2.0.20241003
 types-pyyaml==6.0.12.20240917
 types-requests==2.32.0.20240914
-types-s3transfer==0.10.2
+types-s3transfer==0.10.3
 types-setuptools==75.1.0.20240917
 types-simplejson==3.19.0.20240801
-types-six==1.16.21.20240513
+types-six==1.16.21.20241009
 types-sqlalchemy==1.4.53.34
 types-tabulate==0.9.0.20240106
 types-toml==0.10.8.20240310
@@ -332,5 +336,5 @@ websocket-client==1.8.0
 websockets==13.1
 wheel==0.44.0
 wrapt==1.16.0
-yarl==1.13.1
+yarl==1.14.0
 zipp==3.20.2
diff --git a/pyright/master/requirements-pinned.txt b/pyright/master/requirements-pinned.txt
index bc62e37f2598e..549546fa6b1a3 100644
--- a/pyright/master/requirements-pinned.txt
+++ b/pyright/master/requirements-pinned.txt
@@ -1,9 +1,9 @@
-acryl-datahub==0.14.1
+acryl-datahub==0.14.1.2
 agate==1.9.1
-aiofile==3.8.8
+aiofile==3.9.0
 aiofiles==24.1.0
 aiohappyeyeballs==2.4.3
-aiohttp==3.10.8
+aiohttp==3.10.9
 aiohttp-retry==2.8.3
 aiosignal==1.3.1
 alabaster==1.0.0
@@ -25,8 +25,7 @@ apache-airflow-providers-sqlite==3.8.2
 apeye==1.4.1
 apeye-core==1.1.5
 apispec==6.6.1
-appnope==0.1.4
-argcomplete==3.5.0
+argcomplete==3.5.1
 argon2-cffi==23.1.0
 argon2-cffi-bindings==21.2.0
 arrow==1.3.0
@@ -46,7 +45,7 @@ avro-gen3==0.7.16
 aws-sam-translator==1.91.0
 aws-xray-sdk==2.14.0
 azure-core==1.31.0
-azure-identity==1.18.0
+azure-identity==1.19.0
 azure-storage-blob==12.23.1
 azure-storage-file-datalake==12.17.0
 babel==2.16.0
@@ -57,10 +56,10 @@ billiard==4.2.1
 bleach==6.1.0
 blinker==1.8.2
 bokeh==3.6.0
-boto3==1.35.31
-boto3-stubs-lite==1.35.31
-botocore==1.35.31
-botocore-stubs==1.35.31
+boto3==1.35.36
+boto3-stubs-lite==1.35.36
+botocore==1.35.36
+botocore-stubs==1.35.36
 buildkite-test-collector==0.1.9
 cachecontrol==0.14.0
 cached-property==1.5.2
@@ -72,9 +71,9 @@ cattrs==23.1.2
 celery==5.4.0
 certifi==2024.8.30
 cffi==1.17.1
-cfn-lint==1.15.2
+cfn-lint==1.16.0
 chardet==5.2.0
-charset-normalizer==3.3.2
+charset-normalizer==3.4.0
 click==8.1.7
 click-default-group==1.2.4
 click-didyoumean==0.3.1
@@ -181,7 +180,7 @@ db-dtypes==1.3.0
 dbt-adapters==1.3.2
 dbt-common==1.3.0
 dbt-core==1.8.7
-dbt-duckdb==1.8.4
+dbt-duckdb==1.9.0
 -e examples/experimental/dagster-airlift/examples/dbt-example
 dbt-extractor==0.5.1
 dbt-semantic-interfaces==0.5.1
@@ -198,8 +197,8 @@ dill==0.3.9
 distlib==0.3.8
 distributed==2024.9.1
 distro==1.9.0
-dlt==1.1.0
-dnspython==2.6.1
+dlt==1.2.0
+dnspython==2.7.0
 docker==5.0.3
 docker-image-py==0.1.13
 docker-pycreds==0.4.0
@@ -239,7 +238,7 @@ gitdb==4.0.11
 gitpython==3.1.43
 giturlparse==0.12.0
 google-api-core==2.20.0
-google-api-python-client==2.147.0
+google-api-python-client==2.148.0
 google-auth==2.35.0
 google-auth-httplib2==0.2.0
 google-auth-oauthlib==1.2.1
@@ -256,6 +255,7 @@ graphql-core==3.2.4
 graphql-relay==3.2.0
 graphviz==0.20.3
 great-expectations==0.17.11
+greenlet==3.1.1
 grpcio==1.66.2
 grpcio-health-checking==1.62.3
 grpcio-status==1.62.3
@@ -269,8 +269,8 @@ httplib2==0.22.0
 httptools==0.6.1
 httpx==0.27.2
 humanfriendly==10.0
-humanize==4.10.0
-hypothesis==6.112.2
+humanize==4.11.0
+hypothesis==6.113.0
 idna==3.10
 ijson==3.3.0
 imagesize==1.4.1
@@ -288,7 +288,7 @@ isoduration==20.11.0
 itsdangerous==2.2.0
 jedi==0.19.1
 jinja2==3.1.4
-jiter==0.5.0
+jiter==0.6.1
 jmespath==1.0.1
 joblib==1.4.2
 json5==0.9.25
@@ -316,12 +316,12 @@ kombu==5.4.2
 kopf==1.37.2
 kubernetes==30.1.0
 kubernetes-asyncio==30.1.0
-langchain==0.2.9
+langchain==0.2.10
 langchain-community==0.2.9
 langchain-core==0.2.41
 langchain-openai==0.1.14
 langchain-text-splitters==0.2.4
-langsmith==0.1.130
+langsmith==0.1.132
 lazy-object-proxy==1.10.0
 leather==0.4.0
 limits==3.13.0
@@ -335,7 +335,7 @@ makefun==1.15.6
 mako==1.3.5
 markdown==3.7
 markdown-it-py==3.0.0
-markupsafe==2.1.5
+markupsafe==3.0.1
 marshmallow==3.22.0
 marshmallow-oneofschema==3.1.1
 marshmallow-sqlalchemy==0.26.1
@@ -362,6 +362,7 @@ multimethod==1.10
 mypy-boto3-ecs==1.35.21
 mypy-boto3-emr-serverless==1.35.25
 mypy-boto3-glue==1.35.25
+mypy-boto3-s3==1.35.32
 mypy-extensions==1.0.0
 mypy-protobuf==3.6.0
 mysql-connector-python==9.0.0
@@ -376,13 +377,25 @@ nodeenv==1.9.1
 notebook==7.2.2
 notebook-shim==0.2.4
 numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.6.77
+nvidia-nvtx-cu12==12.1.105
 oauth2client==4.1.3
 oauthlib==3.2.2
 objgraph==3.6.1
 onnx==1.17.0
 onnxconverter-common==1.13.0
 onnxruntime==1.19.2
-openai==1.51.0
+openai==1.51.2
 openapi-schema-validator==0.6.2
 openapi-spec-validator==0.7.1
 opentelemetry-api==1.27.0
@@ -430,6 +443,7 @@ progressbar2==4.5.0
 prometheus-client==0.21.0
 prometheus-flask-exporter==0.23.1
 prompt-toolkit==3.0.48
+propcache==0.2.0
 proto-plus==1.24.0
 protobuf==4.25.5
 psutil==6.0.0
@@ -494,13 +508,13 @@ requirements-parser==0.11.0
 responses==0.23.1
 rfc3339-validator==0.1.4
 rfc3986-validator==0.1.1
-rich==13.9.1
+rich==13.9.2
 rich-argparse==1.5.2
 rpds-py==0.20.0
 rsa==4.9
 ruamel-yaml==0.17.17
-ruff==0.6.8
-s3transfer==0.10.2
+ruff==0.6.9
+s3transfer==0.10.3
 scikit-learn==1.5.2
 scipy==1.14.1
 scrapbook==0.5.0
@@ -509,7 +523,7 @@ seaborn==0.13.2
 selenium==4.25.0
 semver==3.0.2
 send2trash==1.8.3
-sentry-sdk==2.15.0
+sentry-sdk==2.16.0
 setproctitle==1.3.3
 setuptools==70.3.0
 shellingham==1.5.4
@@ -519,7 +533,7 @@ skein==0.8.2
 skl2onnx==1.17.0
 slack-sdk==3.33.1
 sling==1.2.20
-sling-mac-arm64==1.2.20
+sling-linux-amd64==1.2.20
 smmap==5.0.1
 sniffio==1.3.1
 snowballstemmer==2.2.0
@@ -528,10 +542,10 @@ snowflake-sqlalchemy==1.6.1
 sortedcontainers==2.4.0
 soupsieve==2.6
 sphinx==8.0.2
-sphinx-autodoc-typehints==2.4.4
+sphinx-autodoc-typehints==2.5.0
 sphinx-jinja2-compat==0.3.0
 sphinx-prompt==1.9.0
-sphinx-tabs==3.4.5
+sphinx-tabs==3.4.7
 sphinx-toolbox==3.8.0
 sphinxcontrib-applehelp==2.0.0
 sphinxcontrib-devhelp==2.0.0
@@ -542,7 +556,7 @@ sphinxcontrib-serializinghtml==2.0.0
 sqlalchemy==1.4.54
 sqlalchemy-jsonfield==1.0.2
 sqlalchemy-utils==0.41.2
-sqlglot==25.24.1
+sqlglot==25.24.5
 sqlglotrs==0.2.12
 sqlparse==0.5.1
 sshpubkeys==3.3.1
@@ -551,22 +565,22 @@ stack-data==0.6.3
 starlette==0.39.2
 structlog==24.4.0
 sympy==1.13.3
-syrupy==4.7.1
+syrupy==4.7.2
 tableauserverclient==0.25
 tabledata==1.3.3
 tabulate==0.9.0
 tblib==3.0.0
 tenacity==8.5.0
-termcolor==2.4.0
+termcolor==2.5.0
 terminado==0.18.1
 text-unidecode==1.3
 threadpoolctl==3.5.0
-tiktoken==0.7.0
+tiktoken==0.8.0
 tinycss2==1.3.0
 toml==0.10.2
 tomli==2.0.2
 tomlkit==0.13.2
-toolz==0.12.1
+toolz==1.0.0
 toposort==1.10
 torch==2.4.1
 torchvision==0.19.1
@@ -576,9 +590,10 @@ tqdm==4.66.5
 traitlets==5.14.3
 trio==0.26.2
 trio-websocket==0.11.1
+triton==3.0.0
 -e examples/experimental/dagster-airlift/examples/tutorial-example
 -e examples/tutorial_notebook_assets
-twilio==9.3.2
+twilio==9.3.3
 twine==1.15.0
 typeguard==4.3.0
 typepy==1.3.2
@@ -594,14 +609,14 @@ types-mock==5.1.0.20240425
 types-paramiko==3.5.0.20240928
 types-protobuf==5.28.0.20240924
 types-pyopenssl==24.1.0.20240722
-types-python-dateutil==2.9.0.20240906
-types-pytz==2024.2.0.20240913
+types-python-dateutil==2.9.0.20241003
+types-pytz==2024.2.0.20241003
 types-pyyaml==6.0.12.20240917
 types-requests==2.31.0.6
-types-s3transfer==0.10.2
+types-s3transfer==0.10.3
 types-setuptools==75.1.0.20240917
 types-simplejson==3.19.0.20240801
-types-six==1.16.21.20240513
+types-six==1.16.21.20241009
 types-sqlalchemy==1.4.53.34
 types-tabulate==0.9.0.20240106
 types-toml==0.10.8.20240310
@@ -645,6 +660,6 @@ wtforms==3.0.1
 xgboost==2.1.1
 xmltodict==0.12.0
 xyzservices==2024.9.0
-yarl==1.13.1
+yarl==1.14.0
 zict==3.0.0
 zipp==3.20.2
diff --git a/python_modules/libraries/dagster-aws/dagster_aws/pipes/__init__.py b/python_modules/libraries/dagster-aws/dagster_aws/pipes/__init__.py
index 263d3aa48b6d1..3902a5a642f31 100644
--- a/python_modules/libraries/dagster-aws/dagster_aws/pipes/__init__.py
+++ b/python_modules/libraries/dagster-aws/dagster_aws/pipes/__init__.py
@@ -11,6 +11,7 @@
 from dagster_aws.pipes.message_readers import (
     PipesCloudWatchMessageReader,
     PipesLambdaLogsMessageReader,
+    PipesS3LogReader,
     PipesS3MessageReader,
 )
 
@@ -21,6 +22,7 @@
     "PipesS3ContextInjector",
     "PipesLambdaEventContextInjector",
     "PipesS3MessageReader",
+    "PipesS3LogReader",
     "PipesLambdaLogsMessageReader",
     "PipesCloudWatchMessageReader",
     "PipesEMRServerlessClient",
diff --git a/python_modules/libraries/dagster-aws/dagster_aws/pipes/message_readers.py b/python_modules/libraries/dagster-aws/dagster_aws/pipes/message_readers.py
index c8b6df57fade6..fd5ce2117a988 100644
--- a/python_modules/libraries/dagster-aws/dagster_aws/pipes/message_readers.py
+++ b/python_modules/libraries/dagster-aws/dagster_aws/pipes/message_readers.py
@@ -1,9 +1,23 @@
 import base64
+import os
 import random
 import string
 import sys
 from contextlib import contextmanager
-from typing import Any, Dict, Generator, Iterator, List, Optional, Sequence, TextIO, TypedDict
+from typing import (
+    IO,
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Generator,
+    Iterator,
+    List,
+    Optional,
+    Sequence,
+    TextIO,
+    TypedDict,
+)
 
 import boto3
 import dagster._check as check
@@ -13,12 +27,76 @@
 from dagster._core.pipes.context import PipesMessageHandler
 from dagster._core.pipes.utils import (
     PipesBlobStoreMessageReader,
+    PipesChunkedLogReader,
     PipesLogReader,
     extract_message_or_forward_to_file,
     extract_message_or_forward_to_stdout,
 )
 from dagster_pipes import PipesDefaultMessageWriter
 
+if TYPE_CHECKING:
+    from mypy_boto3_s3 import S3Client
+
+
+def _can_read_from_s3(client: "S3Client", bucket: Optional[str], key: Optional[str]):
+    if not bucket or not key:
+        return False
+    else:
+        try:
+            client.head_object(Bucket=bucket, Key=key)
+            return True
+        except ClientError:
+            return False
+
+
+def default_log_decode_fn(contents: bytes) -> str:
+    return contents.decode("utf-8")
+
+
+class PipesS3LogReader(PipesChunkedLogReader):
+    def __init__(
+        self,
+        *,
+        bucket: str,
+        key: str,
+        client: Optional["S3Client"] = None,
+        interval: float = 10,
+        target_stream: Optional[IO[str]] = None,
+        # TODO: maybe move this parameter to a different scope
+        decode_fn: Optional[Callable[[bytes], str]] = None,
+        debug_info: Optional[str] = None,
+    ):
+        self.bucket = bucket
+        self.key = key
+        self.client: "S3Client" = client or boto3.client("s3")
+        self.decode_fn = decode_fn or default_log_decode_fn
+
+        self.log_position = 0
+
+        super().__init__(
+            interval=interval, target_stream=target_stream or sys.stdout, debug_info=debug_info
+        )
+
+    @property
+    def name(self) -> str:
+        return f"PipesS3LogReader(s3://{os.path.join(self.bucket, self.key)})"
+
+    def target_is_readable(self, params: PipesParams) -> bool:
+        return _can_read_from_s3(
+            client=self.client,
+            bucket=self.bucket,
+            key=self.key,
+        )
+
+    def download_log_chunk(self, params: PipesParams) -> Optional[str]:
+        text = self.decode_fn(
+            self.client.get_object(Bucket=self.bucket, Key=self.key)["Body"].read()
+        )
+        current_position = self.log_position
+        self.log_position += len(text)
+
+        return text[current_position:]
+
 
 class PipesS3MessageReader(PipesBlobStoreMessageReader):
     """Message reader that reads messages by periodically reading message chunks from a specified S3
diff --git a/python_modules/libraries/dagster-aws/dagster_aws_tests/pipes_tests/test_pipes.py b/python_modules/libraries/dagster-aws/dagster_aws_tests/pipes_tests/test_pipes.py
index f2f0995bb88c0..34b8396fabaf9 100644
--- a/python_modules/libraries/dagster-aws/dagster_aws_tests/pipes_tests/test_pipes.py
+++ b/python_modules/libraries/dagster-aws/dagster_aws_tests/pipes_tests/test_pipes.py
@@ -10,6 +10,7 @@
 import time
 from contextlib import contextmanager
 from tempfile import NamedTemporaryFile
+from threading import Event
 from typing import TYPE_CHECKING, Any, Callable, Iterator, Literal, Tuple
 from uuid import uuid4
 
@@ -38,6 +39,7 @@
     PipesLambdaClient,
     PipesLambdaLogsMessageReader,
     PipesS3ContextInjector,
+    PipesS3LogReader,
     PipesS3MessageReader,
 )
 from dagster_aws_tests.pipes_tests.fake_ecs import LocalECSMockClient
@@ -152,6 +154,35 @@ def s3_client(moto_server):
     return client
 
 
+def test_s3_log_reader(s3_client, capsys):
+    key = str(uuid4())
+    log_reader = PipesS3LogReader(client=s3_client, bucket=_S3_TEST_BUCKET, key=key)
+    is_session_closed = Event()
+
+    assert not log_reader.target_is_readable({})
+
+    s3_client.put_object(Bucket=_S3_TEST_BUCKET, Key=key, Body=b"Line 0\nLine 1")
+
+    assert log_reader.target_is_readable({})
+
+    log_reader.start({}, is_session_closed)
+    assert log_reader.is_running()
+
+    s3_client.put_object(Bucket=_S3_TEST_BUCKET, Key=key, Body=b"Line 0\nLine 1\nLine 2")
+
+    is_session_closed.set()
+
+    log_reader.stop()
+
+    assert not log_reader.is_running()
+
+    captured = capsys.readouterr()
+
+    assert captured.out == "Line 0\nLine 1\nLine 2"
+
+    assert sys.stdout is not None
+
+
 def test_s3_pipes_components(
     capsys,
     tmpdir,
diff --git a/python_modules/libraries/dagster-aws/setup.py b/python_modules/libraries/dagster-aws/setup.py
index 3616471fc37bf..85ccfd520abc7 100644
--- a/python_modules/libraries/dagster-aws/setup.py
+++ b/python_modules/libraries/dagster-aws/setup.py
@@ -45,7 +45,7 @@ def get_version() -> str:
         "redshift": ["psycopg2-binary"],
         "pyspark": ["dagster-pyspark"],
         "stubs": [
-            "boto3-stubs-lite[ecs,glue,emr-serverless]",
+            "boto3-stubs-lite[ecs,glue,emr-serverless,s3]",
         ],
         "test": [
             "botocore!=1.32.1",