From eabffed162a3331d8c400127e68d3a83c257155c Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Tue, 19 Jul 2022 12:28:13 +0200 Subject: [PATCH 01/10] run tests with both backends --- test/runtests.jl | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 92dc42fa..b5176a0b 100755 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -51,8 +51,18 @@ include("awss3.jl") # creates `s3path_tests(config)` # Set `AWSConfig` as the default for the following tests aws = global_aws_config(AWSConfig()) - @testset "S3" begin - awss3_tests(aws) - s3path_tests(aws) + + prev_backend = AWS.DownloadsBackend[] + for backend in (AWS.HTTPBackend, AWS.DownloadsBackend) + AWS.DEFAULT_BACKEND[] = backend() + try + @testset "S3 with $backend" begin + awss3_tests(aws) + s3path_tests(aws) + end + finally + AWS.DEFAULT_BACKEND[] = prev_backend + end end + end From 81bdd269c661307b30fc0d8bbd7c5ab6c7c7eca3 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Tue, 19 Jul 2022 12:28:51 +0200 Subject: [PATCH 02/10] fix typo --- test/runtests.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index b5176a0b..b2ec12a2 100755 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -51,8 +51,8 @@ include("awss3.jl") # creates `s3path_tests(config)` # Set `AWSConfig` as the default for the following tests aws = global_aws_config(AWSConfig()) - - prev_backend = AWS.DownloadsBackend[] + + prev_backend = AWS.DEFAULT_BACKEND[] for backend in (AWS.HTTPBackend, AWS.DownloadsBackend) AWS.DEFAULT_BACKEND[] = backend() try From 0a65904adac0d36d51226daf42010ececc513547 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Tue, 19 Jul 2022 12:30:24 +0200 Subject: [PATCH 03/10] Update test/runtests.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- test/runtests.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index b2ec12a2..8245bbd1 100755 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -64,5 +64,4 @@ include("awss3.jl") # creates `s3path_tests(config)` AWS.DEFAULT_BACKEND[] = prev_backend end end - end From 49b30df3434ec935142efb17913c1a102b74258d Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Tue, 19 Jul 2022 12:42:44 +0200 Subject: [PATCH 04/10] add warning --- src/AWSS3.jl | 3 +++ test/awss3.jl | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/AWSS3.jl b/src/AWSS3.jl index c6129b2d..13362bad 100644 --- a/src/AWSS3.jl +++ b/src/AWSS3.jl @@ -217,6 +217,9 @@ end [HEAD Object](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html) Retrieves metadata from an object without returning the object itself. + +!!! warning + This function returns the headers from `S3.head_object` as `Dict` of key-value pairs. [HTTP/1.1 headers are case insensitive](https://datatracker.ietf.org/doc/html/rfc2616#section-4.2) while [HTTP/2 headers must be lowercase](https://datatracker.ietf.org/doc/html/rfc7540#section-8.1.2). Therefore, the casing of the keys of the returned dict may depend on the HTTP client implementation (e.g. via `AWS.DownloadsBackend` vs `AWS.HTTPBackend`). """ function s3_get_meta( aws::AbstractAWSConfig, bucket, path; version::AbstractS3Version=nothing, kwargs... diff --git a/test/awss3.jl b/test/awss3.jl index 526a212a..4f005062 100644 --- a/test/awss3.jl +++ b/test/awss3.jl @@ -103,7 +103,7 @@ function awss3_tests(config) @testset "default Content-Type" begin # https://github.com/samoconnor/AWSS3.jl/issues/24 - ctype(key) = s3_get_meta(bucket_name, key)["Content-Type"] + ctype(key) = AWSS3.get_robust_case(s3_get_meta(bucket_name, key), "Content-Type") for k in ["file.foo", "file", "file_html", "file.d/html", "foobar.html/file.htm"] is_aws(config) && k == "file" && continue From efbaa2b056b1b2acfbba388958c2f45ad146772a Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Tue, 19 Jul 2022 14:28:45 +0200 Subject: [PATCH 05/10] print error --- test/awss3.jl | 2 +- test/s3path.jl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/awss3.jl b/test/awss3.jl index 4f005062..fa09ad6b 100644 --- a/test/awss3.jl +++ b/test/awss3.jl @@ -98,7 +98,7 @@ function awss3_tests(config) @testset "Check Metadata" begin meta = s3_get_meta(config, bucket_name, "key1") - @test meta["ETag"] == "\"68bc8898af64159b72f349b391a7ae35\"" + @test get_robust_case(meta, "ETag") == "\"68bc8898af64159b72f349b391a7ae35\"" end @testset "default Content-Type" begin diff --git a/test/s3path.jl b/test/s3path.jl index 7c082842..26c7c1a3 100644 --- a/test/s3path.jl +++ b/test/s3path.jl @@ -631,6 +631,7 @@ function s3path_tests(config) alt_region = prev_config.region == "us-east-2" ? "us-east-1" : "us-east-2" try global_aws_config(; region=alt_region) # this is the wrong region! + read(path, String) @test_throws AWS.AWSException read(path, String) # restore the right region From ca01dc9eee958c4506bb6ad2d55f6b4067c994e1 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Tue, 19 Jul 2022 14:41:11 +0200 Subject: [PATCH 06/10] make clearer --- test/awss3.jl | 2 +- test/s3path.jl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/awss3.jl b/test/awss3.jl index fa09ad6b..919de251 100644 --- a/test/awss3.jl +++ b/test/awss3.jl @@ -98,7 +98,7 @@ function awss3_tests(config) @testset "Check Metadata" begin meta = s3_get_meta(config, bucket_name, "key1") - @test get_robust_case(meta, "ETag") == "\"68bc8898af64159b72f349b391a7ae35\"" + @test AWSS3.get_robust_case(meta, "ETag") == "\"68bc8898af64159b72f349b391a7ae35\"" end @testset "default Content-Type" begin diff --git a/test/s3path.jl b/test/s3path.jl index 26c7c1a3..13ddfa3a 100644 --- a/test/s3path.jl +++ b/test/s3path.jl @@ -631,6 +631,7 @@ function s3path_tests(config) alt_region = prev_config.region == "us-east-2" ? "us-east-1" : "us-east-2" try global_aws_config(; region=alt_region) # this is the wrong region! + println("Backend: $(AWS.DEFAULT_BACKEND[])") read(path, String) @test_throws AWS.AWSException read(path, String) From 967ca0dea3d34c626bb5029a6d4bbcc581946437 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Tue, 19 Jul 2022 14:47:41 +0200 Subject: [PATCH 07/10] workaround AWS.jl issue --- test/s3path.jl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/s3path.jl b/test/s3path.jl index 13ddfa3a..232cb1e3 100644 --- a/test/s3path.jl +++ b/test/s3path.jl @@ -631,9 +631,13 @@ function s3path_tests(config) alt_region = prev_config.region == "us-east-2" ? "us-east-1" : "us-east-2" try global_aws_config(; region=alt_region) # this is the wrong region! - println("Backend: $(AWS.DEFAULT_BACKEND[])") - read(path, String) - @test_throws AWS.AWSException read(path, String) + + if AWS.DEFAULT_BACKEND[] isa AWS.DownloadsBackend + # https://github.com/JuliaCloud/AWS.jl/issues/564 + @test_broken AWS.AWSException read(path, String) + else + @test_throws AWS.AWSException read(path, String) + end # restore the right region global_aws_config(prev_config) From 043844700b766a4e5c1af8ea75a557ed56c122d4 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Tue, 19 Jul 2022 14:49:19 +0200 Subject: [PATCH 08/10] add newlines, fix spacing --- src/AWSS3.jl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/AWSS3.jl b/src/AWSS3.jl index 13362bad..1866786d 100644 --- a/src/AWSS3.jl +++ b/src/AWSS3.jl @@ -212,14 +212,18 @@ function s3_get_file( end """ - s3_get_meta([::AbstractAWSConfig], bucket, path; [version=], kwargs...) + s3_get_meta([::AbstractAWSConfig], bucket, path; [version=], kwargs...) [HEAD Object](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html) Retrieves metadata from an object without returning the object itself. !!! warning - This function returns the headers from `S3.head_object` as `Dict` of key-value pairs. [HTTP/1.1 headers are case insensitive](https://datatracker.ietf.org/doc/html/rfc2616#section-4.2) while [HTTP/2 headers must be lowercase](https://datatracker.ietf.org/doc/html/rfc7540#section-8.1.2). Therefore, the casing of the keys of the returned dict may depend on the HTTP client implementation (e.g. via `AWS.DownloadsBackend` vs `AWS.HTTPBackend`). + This function returns the headers from `S3.head_object` as `Dict` of key-value pairs. + [HTTP/1.1 headers are case insensitive](https://datatracker.ietf.org/doc/html/rfc2616#section-4.2) + while [HTTP/2 headers must be lowercase](https://datatracker.ietf.org/doc/html/rfc7540#section-8.1.2). + Therefore, the casing of the keys of the returned dict may depend on the HTTP client implementation + (e.g. via `AWS.DownloadsBackend` vs `AWS.HTTPBackend`). """ function s3_get_meta( aws::AbstractAWSConfig, bucket, path; version::AbstractS3Version=nothing, kwargs... From e1eb51cb7d09a8f6a65af47605d4b2d6f81c8544 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Tue, 19 Jul 2022 14:49:30 +0200 Subject: [PATCH 09/10] Update test/s3path.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- test/s3path.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/s3path.jl b/test/s3path.jl index 232cb1e3..d330d6c1 100644 --- a/test/s3path.jl +++ b/test/s3path.jl @@ -631,7 +631,6 @@ function s3path_tests(config) alt_region = prev_config.region == "us-east-2" ? "us-east-1" : "us-east-2" try global_aws_config(; region=alt_region) # this is the wrong region! - if AWS.DEFAULT_BACKEND[] isa AWS.DownloadsBackend # https://github.com/JuliaCloud/AWS.jl/issues/564 @test_broken AWS.AWSException read(path, String) From 1b5b7ada92ae2388b04c278d737655c8fc22fa81 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Tue, 19 Jul 2022 15:37:39 +0200 Subject: [PATCH 10/10] awkward test-throws + test-broken --- test/s3path.jl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/s3path.jl b/test/s3path.jl index 232cb1e3..8b46b2f6 100644 --- a/test/s3path.jl +++ b/test/s3path.jl @@ -631,10 +631,15 @@ function s3path_tests(config) alt_region = prev_config.region == "us-east-2" ? "us-east-1" : "us-east-2" try global_aws_config(; region=alt_region) # this is the wrong region! - + if AWS.DEFAULT_BACKEND[] isa AWS.DownloadsBackend # https://github.com/JuliaCloud/AWS.jl/issues/564 - @test_broken AWS.AWSException read(path, String) + local e + try + read(path, String) + catch e + end + @test_broken e isa AWS.AWSException else @test_throws AWS.AWSException read(path, String) end