Skip to content

Commit

Permalink
[dagster-aws, dagster-pipes] Add PipesS3LogReader (#25224)
Browse files Browse the repository at this point in the history
## Summary & Motivation

Add `PipesS3LogReader` which can read logs from known S3 objects. 

## How I Tested These Changes

Added a test
  • Loading branch information
danielgafni authored Oct 14, 2024
1 parent a32d733 commit 169d673
Show file tree
Hide file tree
Showing 6 changed files with 197 additions and 67 deletions.
52 changes: 28 additions & 24 deletions pyright/alt-1/requirements-pinned.txt
Original file line number Diff line number Diff line change
@@ -1,21 +1,20 @@
agate==1.9.1
aiobotocore==2.15.1
aiofile==3.8.8
aiofile==3.9.0
aiohappyeyeballs==2.4.3
aiohttp==3.10.8
aiohttp==3.10.9
aioitertools==0.12.0
aiosignal==1.3.1
alembic==1.13.3
aniso8601==9.0.1
annotated-types==0.7.0
anyio==4.6.0
appnope==0.1.4
argon2-cffi==23.1.0
argon2-cffi-bindings==21.2.0
arrow==1.3.0
asn1crypto==1.5.1
-e examples/assets_pandas_type_metadata
astroid==3.3.4
astroid==3.3.5
asttokens==2.4.1
async-lru==2.0.4
attrs==24.2.0
Expand All @@ -25,16 +24,16 @@ backports-tarfile==1.2.0
beautifulsoup4==4.12.3
bleach==6.1.0
boto3==1.35.23
boto3-stubs-lite==1.35.31
boto3-stubs-lite==1.35.36
botocore==1.35.23
botocore-stubs==1.35.31
botocore-stubs==1.35.36
buildkite-test-collector==0.1.9
cachetools==5.5.0
caio==0.9.17
certifi==2024.8.30
cffi==1.17.1
chardet==5.2.0
charset-normalizer==3.3.2
charset-normalizer==3.4.0
click==8.1.7
colorama==0.4.6
coloredlogs==14.0
Expand Down Expand Up @@ -69,7 +68,7 @@ db-dtypes==1.3.0
dbt-adapters==1.3.2
dbt-common==1.3.0
dbt-core==1.8.7
dbt-duckdb==1.8.4
dbt-duckdb==1.9.0
dbt-extractor==0.5.1
dbt-semantic-interfaces==0.5.1
dbt-snowflake==1.8.3
Expand All @@ -78,7 +77,7 @@ decopatch==1.4.10
decorator==5.1.1
deepdiff==8.0.1
defusedxml==0.7.1
deltalake==0.20.1
deltalake==0.20.2
dill==0.3.9
distlib==0.3.8
docker==7.1.0
Expand All @@ -94,7 +93,7 @@ frozenlist==1.4.1
fsspec==2024.3.0
gcsfs==0.8.0
google-api-core==2.20.0
google-api-python-client==2.147.0
google-api-python-client==2.148.0
google-auth==2.35.0
google-auth-httplib2==0.2.0
google-auth-oauthlib==1.2.1
Expand All @@ -108,6 +107,7 @@ gql==3.5.0
graphene==3.3
graphql-core==3.2.4
graphql-relay==3.2.0
greenlet==3.1.1
grpcio==1.66.2
grpcio-health-checking==1.62.3
grpcio-status==1.62.3
Expand All @@ -118,7 +118,7 @@ httplib2==0.22.0
httptools==0.6.1
httpx==0.27.2
humanfriendly==10.0
hypothesis==6.112.2
hypothesis==6.113.0
idna==3.10
importlib-metadata==6.11.0
iniconfig==2.0.0
Expand All @@ -131,13 +131,14 @@ jaraco-classes==3.4.0
jaraco-context==6.0.1
jaraco-functools==4.1.0
jedi==0.19.1
jeepney==0.8.0
jinja2==3.1.4
jmespath==1.0.1
joblib==1.4.2
json5==0.9.25
jsonpointer==3.0.0
jsonschema==4.23.0
jsonschema-specifications==2023.12.1
jsonschema-specifications==2024.10.1
jupyter-client==8.6.3
jupyter-core==5.7.2
jupyter-events==0.10.0
Expand All @@ -154,7 +155,7 @@ logbook==1.5.3
makefun==1.15.6
mako==1.3.5
markdown-it-py==3.0.0
markupsafe==2.1.5
markupsafe==3.0.1
mashumaro==3.13.1
matplotlib==3.9.2
matplotlib-inline==0.1.7
Expand All @@ -172,6 +173,7 @@ mypy==1.11.2
mypy-boto3-ecs==1.35.21
mypy-boto3-emr-serverless==1.35.25
mypy-boto3-glue==1.35.25
mypy-boto3-s3==1.35.32
mypy-extensions==1.0.0
mypy-protobuf==3.6.0
nbclient==0.10.0
Expand All @@ -181,7 +183,7 @@ nest-asyncio==1.6.0
networkx==3.3
nodeenv==1.9.1
notebook-shim==0.2.4
numpy==2.1.1
numpy==2.1.2
oauth2client==4.1.3
oauthlib==3.2.2
objgraph==3.6.1
Expand All @@ -205,6 +207,7 @@ polars==1.9.0
-e examples/project_fully_featured
prometheus-client==0.21.0
prompt-toolkit==3.0.48
propcache==0.2.0
proto-plus==1.24.0
protobuf==4.25.5
psutil==6.0.0
Expand Down Expand Up @@ -249,14 +252,15 @@ requests-toolbelt==1.0.0
responses==0.23.1
rfc3339-validator==0.1.4
rfc3986-validator==0.1.1
rich==13.9.1
rich==13.9.2
rpds-py==0.20.0
rsa==4.9
s3fs==2024.3.0
s3transfer==0.10.2
s3transfer==0.10.3
scikit-learn==1.5.2
scipy==1.14.1
seaborn==0.13.2
secretstorage==3.3.3
send2trash==1.8.3
setuptools==75.1.0
shellingham==1.5.4
Expand All @@ -268,13 +272,13 @@ snowflake-sqlalchemy==1.5.1
sortedcontainers==2.4.0
soupsieve==2.6
sqlalchemy==1.4.54
sqlglot==25.24.1
sqlglot==25.24.5
sqlglotrs==0.2.12
sqlparse==0.5.1
stack-data==0.6.3
starlette==0.39.2
structlog==24.4.0
syrupy==4.7.1
syrupy==4.7.2
tabulate==0.9.0
terminado==0.18.1
text-unidecode==1.3
Expand All @@ -284,7 +288,7 @@ tomli==2.0.2
tomlkit==0.13.2
toposort==1.10
tornado==6.4.1
tox==4.21.0
tox==4.21.2
tqdm==4.66.5
traitlets==5.14.3
typeguard==4.3.0
Expand All @@ -300,14 +304,14 @@ types-mock==5.1.0.20240425
types-paramiko==3.5.0.20240928
types-protobuf==5.28.0.20240924
types-pyopenssl==24.1.0.20240722
types-python-dateutil==2.9.0.20240906
types-pytz==2024.2.0.20240913
types-python-dateutil==2.9.0.20241003
types-pytz==2024.2.0.20241003
types-pyyaml==6.0.12.20240917
types-requests==2.32.0.20240914
types-s3transfer==0.10.2
types-s3transfer==0.10.3
types-setuptools==75.1.0.20240917
types-simplejson==3.19.0.20240801
types-six==1.16.21.20240513
types-six==1.16.21.20241009
types-sqlalchemy==1.4.53.34
types-tabulate==0.9.0.20240106
types-toml==0.10.8.20240310
Expand All @@ -332,5 +336,5 @@ websocket-client==1.8.0
websockets==13.1
wheel==0.44.0
wrapt==1.16.0
yarl==1.13.1
yarl==1.14.0
zipp==3.20.2
Loading

0 comments on commit 169d673

Please sign in to comment.