diff --git a/build.gradle.kts b/build.gradle.kts index 5afe5d29..f826db96 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -63,6 +63,7 @@ dependencies { testImplementation("org.mockito:mockito-junit-jupiter:3.9.0") testImplementation("org.testcontainers:testcontainers:1.18.1") testImplementation("org.testcontainers:testcontainers:1.18.1") + testImplementation("org.assertj:assertj-core:3.24.2") } configure { diff --git a/docs/content/reference/configuration-options.md b/docs/content/reference/configuration-options.md index 0171a4ac..13c058d5 100644 --- a/docs/content/reference/configuration-options.md +++ b/docs/content/reference/configuration-options.md @@ -4,26 +4,27 @@ A complete list of environment variables which can be set to configure the client. -| Key | Default | Description | -| ------------------------------------------| ----------- |---------------------------------------------------------------------------------------------------------------------------------------- | -| s3fs.access.key | none | AWS access key, used to identify the user interacting with AWS | -| s3fs.secret.key | none | AWS secret access key, used to authenticate the user interacting with AWS | -| s3fs.request.metric.collector.class | TODO | Fully-qualified class name to instantiate an AWS SDK request/response metric collector | -| s3fs.connection.timeout | TODO | Timeout (in milliseconds) for establishing a connection to a remote service | -| s3fs.max.connections | TODO | Maximum number of connections allowed in a connection pool | -| s3fs.max.retry.error | TODO | Maximum number of times that a single request should be retried, assuming it fails for a retryable error | -| s3fs.protocol | TODO | Protocol (HTTP or HTTPS) to use when connecting to AWS | -| s3fs.proxy.domain | none | For NTLM proxies: The Windows domain name to use when authenticating with the proxy | -| s3fs.proxy.host | none | Proxy host name either from the configured endpoint or from the "http.proxyHost" system property | -| s3fs.proxy.password | none | The password to use when connecting through a proxy | -| s3fs.proxy.port | none | Proxy port either from the configured endpoint or from the "http.proxyPort" system property | -| s3fs.proxy.username | none | The username to use when connecting through a proxy | -| s3fs.proxy.workstation | none | For NTLM proxies: The Windows workstation name to use when authenticating with the proxy | -| s3fs.region | none | The AWS Region to configure the client | -| s3fs.socket.send.buffer.size.hint | TODO | The size hint (in bytes) for the low level TCP send buffer | -| s3fs.socket.receive.buffer.size.hint | TODO | The size hint (in bytes) for the low level TCP receive buffer | -| s3fs.socket.timeout | TODO | Timeout (in milliseconds) for each read to the underlying socket | -| s3fs.user.agent.prefix | TODO | Prefix of the user agent that is sent with each request to AWS | -| s3fs.amazon.s3.factory.class | TODO | Fully-qualified class name to instantiate a S3 factory base class which creates a S3 client instance | -| s3fs.signer.override | TODO | Fully-qualified class name to define the signer that should be used when authenticating with AWS | -| s3fs.path.style.access | TODO | Boolean that indicates whether the client uses path-style access for all requests | +| Key | Default | Description | +| ------------------------------------------|---------|-------------------------------------------------------------------------------------------------------------------------| +| s3fs.access.key | none | AWS access key, used to identify the user interacting with AWS | +| s3fs.secret.key | none | AWS secret access key, used to authenticate the user interacting with AWS | +| s3fs.request.metric.collector.class | TODO | Fully-qualified class name to instantiate an AWS SDK request/response metric collector | +| s3fs.connection.timeout | TODO | Timeout (in milliseconds) for establishing a connection to a remote service | +| s3fs.max.connections | TODO | Maximum number of connections allowed in a connection pool | +| s3fs.max.retry.error | TODO | Maximum number of times that a single request should be retried, assuming it fails for a retryable error | +| s3fs.protocol | TODO | Protocol (HTTP or HTTPS) to use when connecting to AWS | +| s3fs.proxy.domain | none | For NTLM proxies: The Windows domain name to use when authenticating with the proxy | +| s3fs.proxy.host | none | Proxy host name either from the configured endpoint or from the "http.proxyHost" system property | +| s3fs.proxy.password | none | The password to use when connecting through a proxy | +| s3fs.proxy.port | none | Proxy port either from the configured endpoint or from the "http.proxyPort" system property | +| s3fs.proxy.username | none | The username to use when connecting through a proxy | +| s3fs.proxy.workstation | none | For NTLM proxies: The Windows workstation name to use when authenticating with the proxy | +| s3fs.region | none | The AWS Region to configure the client | +| s3fs.socket.send.buffer.size.hint | TODO | The size hint (in bytes) for the low level TCP send buffer | +| s3fs.socket.receive.buffer.size.hint | TODO | The size hint (in bytes) for the low level TCP receive buffer | +| s3fs.socket.timeout | TODO | Timeout (in milliseconds) for each read to the underlying socket | +| s3fs.user.agent.prefix | TODO | Prefix of the user agent that is sent with each request to AWS | +| s3fs.amazon.s3.factory.class | TODO | Fully-qualified class name to instantiate a S3 factory base class which creates a S3 client instance | +| s3fs.signer.override | TODO | Fully-qualified class name to define the signer that should be used when authenticating with AWS | +| s3fs.path.style.access | TODO | Boolean that indicates whether the client uses path-style access for all requests | +| s3fs.request.header.cache-control | blank | Configures the `cacheControl` on request builders (i.e. `CopyObjectRequest`, `PutObjectRequest`, etc) | diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3Factory.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3Factory.java index a0f1ece7..e3757cce 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/S3Factory.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3Factory.java @@ -21,7 +21,6 @@ import software.amazon.awssdk.http.apache.ProxyConfiguration; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.regions.providers.AwsRegionProvider; -import software.amazon.awssdk.regions.providers.AwsRegionProviderChain; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3ClientBuilder; import software.amazon.awssdk.services.s3.S3Configuration; @@ -84,6 +83,8 @@ public abstract class S3Factory public static final String PATH_STYLE_ACCESS = "s3fs.path.style.access"; + public static final String REQUEST_HEADER_CACHE_CONTROL = "s3fs.request.header.cache-control"; + private static final Logger LOGGER = LoggerFactory.getLogger(S3Factory.class); private static final String DEFAULT_PROTOCOL = Protocol.HTTPS.toString(); diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileChannel.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileChannel.java index e539e11f..63bde8e5 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileChannel.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileChannel.java @@ -15,9 +15,7 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -67,6 +65,15 @@ public class S3FileChannel */ private final Lock writeReadChannelLock = readWriteLock.readLock(); + public S3FileChannel(final S3Path path, + final Set options, + final ExecutorService executor, + final boolean tempFileRequired) + throws IOException + { + this(path, options, executor, tempFileRequired, new HashMap<>()); + } + /** * Open or creates a file, returning a file channel. * @@ -79,13 +86,15 @@ public class S3FileChannel public S3FileChannel(final S3Path path, final Set options, final ExecutorService executor, - final boolean tempFileRequired) + final boolean tempFileRequired, + final Map properties) throws IOException { openCloseLock.lock(); this.path = path; this.options = Collections.unmodifiableSet(new HashSet<>(options)); + String headerCacheControlProperty = path.getFileSystem().getRequestHeaderCacheControlProperty(); boolean exists = path.getFileSystem().provider().exists(path); boolean removeTempFile = false; diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystem.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystem.java index 940940d4..23f776d7 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystem.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystem.java @@ -7,6 +7,7 @@ import java.nio.file.PathMatcher; import java.nio.file.WatchService; import java.nio.file.attribute.UserPrincipalLookupService; +import java.util.Properties; import java.util.Set; import com.google.common.collect.ImmutableList; @@ -35,16 +36,28 @@ public class S3FileSystem private final int cache; + private final Properties properties; + public S3FileSystem(final S3FileSystemProvider provider, final String key, final S3Client client, - final String endpoint) + final String endpoint, + Properties properties) { this.provider = provider; this.key = key; this.client = client; this.endpoint = endpoint; this.cache = 60000; // 1 minute cache for the s3Path + this.properties = properties; + } + + public S3FileSystem(final S3FileSystemProvider provider, + final String key, + final S3Client client, + final String endpoint) + { + this(provider, key, client, endpoint, new Properties()); } @Override @@ -172,6 +185,20 @@ public String[] key2Parts(String keyParts) return split; } + public int getCache() + { + return cache; + } + + + /** + * @return The value of the {@link S3Factory#REQUEST_HEADER_CACHE_CONTROL} property. Default is empty. + */ + public String getRequestHeaderCacheControlProperty() + { + return properties.getProperty(S3Factory.REQUEST_HEADER_CACHE_CONTROL, ""); // default is nothing. + } + @Override public int hashCode() { @@ -230,8 +257,4 @@ public int compareTo(final S3FileSystem o) return key.compareTo(o.getKey()); } - public int getCache() - { - return cache; - } } diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java index c3335782..82922b8f 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3FileSystemProvider.java @@ -597,7 +597,10 @@ public OutputStream newOutputStream(final Path path, final Map metadata = buildMetadataFromPath(path); - return new S3OutputStream(s3Path.getFileSystem().getClient(), s3Path.toS3ObjectId(), metadata); + + S3FileSystem fileSystem = s3Path.getFileSystem(); + + return new S3OutputStream(fileSystem.getClient(), s3Path.toS3ObjectId(), null, metadata, fileSystem.getRequestHeaderCacheControlProperty()); } private void validateCreateAndTruncateOptions(final Path path, @@ -696,6 +699,7 @@ public void createDirectory(Path dir, final PutObjectRequest request = PutObjectRequest.builder() .bucket(bucketName) .key(directoryKey) + .cacheControl(s3Path.getFileSystem().getRequestHeaderCacheControlProperty()) .contentLength(0L) .build(); @@ -869,6 +873,7 @@ public void copy(Path source, final CopyObjectRequest request = CopyObjectRequest.builder() .copySource(encodedUrl) + .cacheControl(s3Target.getFileSystem().getRequestHeaderCacheControlProperty()) .destinationBucket(bucketNameTarget) .destinationKey(keyTarget) .build(); @@ -1097,7 +1102,8 @@ public S3FileSystem createFileSystem(URI uri, final String key = getFileSystemKey(uri, props); final S3Client client = getS3Client(uri, props); final String host = uri.getHost(); - return new S3FileSystem(this, key, client, host); + final Properties properties = new Properties(props); + return new S3FileSystem(this, key, client, host, properties); } protected S3Client getS3Client(URI uri, diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3OutputStream.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3OutputStream.java index 0e28d163..33f3c402 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/S3OutputStream.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3OutputStream.java @@ -100,6 +100,8 @@ public final class S3OutputStream */ private List partETags; + private final String requestCacheControlHeader; + /** * Creates a new {@code S3OutputStream} that writes data directly into the S3 object with the given {@code objectId}. * No special object metadata or storage class will be attached to the object. @@ -115,6 +117,7 @@ public S3OutputStream(final S3Client s3Client, this.objectId = requireNonNull(objectId); this.metadata = new HashMap<>(); this.storageClass = null; + this.requestCacheControlHeader = ""; } /** @@ -132,8 +135,9 @@ public S3OutputStream(final S3Client s3Client, { this.s3Client = requireNonNull(s3Client); this.objectId = requireNonNull(objectId); - this.storageClass = storageClass; this.metadata = new HashMap<>(); + this.storageClass = storageClass; + this.requestCacheControlHeader = ""; } /** @@ -154,6 +158,7 @@ public S3OutputStream(final S3Client s3Client, this.objectId = requireNonNull(objectId); this.storageClass = null; this.metadata = new HashMap<>(metadata); + this.requestCacheControlHeader = ""; } /** @@ -175,6 +180,31 @@ public S3OutputStream(final S3Client s3Client, this.objectId = requireNonNull(objectId); this.storageClass = storageClass; this.metadata = new HashMap<>(metadata); + this.requestCacheControlHeader = ""; + } + + /** + * Creates a new {@code S3OutputStream} that writes data directly into the S3 object with the given {@code objectId}. + * The given {@code metadata} will be attached to the written object. + * + * @param s3Client S3 ClientAPI to use + * @param objectId ID of the S3 object to store data into + * @param storageClass S3 Client storage class to apply to the newly created S3 object, if any + * @param metadata metadata to attach to the written object + * @param requestCacheControlHeader Controls + * @throws NullPointerException if at least one parameter except {@code storageClass} is {@code null} + */ + public S3OutputStream(final S3Client s3Client, + final S3ObjectId objectId, + final StorageClass storageClass, + final Map metadata, + final String requestCacheControlHeader) + { + this.s3Client = requireNonNull(s3Client); + this.objectId = requireNonNull(objectId); + this.storageClass = storageClass; + this.metadata = new HashMap<>(metadata); + this.requestCacheControlHeader = requestCacheControlHeader; } //protected for testing purposes @@ -435,6 +465,7 @@ private void putObject(final long contentLength, final PutObjectRequest.Builder requestBuilder = PutObjectRequest.builder() .bucket(objectId.getBucket()) .key(objectId.getKey()) + .cacheControl(requestCacheControlHeader) .contentLength(contentLength) .contentType(contentType) .metadata(metadataMap); diff --git a/src/main/java/org/carlspring/cloud/storage/s3fs/S3SeekableByteChannel.java b/src/main/java/org/carlspring/cloud/storage/s3fs/S3SeekableByteChannel.java index 60ca58d6..51b3f4b7 100644 --- a/src/main/java/org/carlspring/cloud/storage/s3fs/S3SeekableByteChannel.java +++ b/src/main/java/org/carlspring/cloud/storage/s3fs/S3SeekableByteChannel.java @@ -35,6 +35,8 @@ public class S3SeekableByteChannel private final Path tempFile; + private final String requestCacheControlHeader; + /** * Open or creates a file, returning a seekable byte channel @@ -51,6 +53,7 @@ public S3SeekableByteChannel(final S3Path path, { this.path = path; this.options = Collections.unmodifiableSet(new HashSet<>(options)); + this.requestCacheControlHeader = path.getFileSystem().getRequestHeaderCacheControlProperty(); final String key = path.getKey(); final boolean exists = path.getFileSystem().provider().exists(path); @@ -176,6 +179,7 @@ protected void sync() builder.bucket(path.getFileStore().name()); builder.key(path.getKey()); + builder.cacheControl(requestCacheControlHeader); S3Client client = path.getFileSystem().getClient(); diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/BaseIntegrationTest.java b/src/test/java/org/carlspring/cloud/storage/s3fs/BaseIntegrationTest.java index 0f189790..5b2a01d6 100644 --- a/src/test/java/org/carlspring/cloud/storage/s3fs/BaseIntegrationTest.java +++ b/src/test/java/org/carlspring/cloud/storage/s3fs/BaseIntegrationTest.java @@ -1,10 +1,16 @@ package org.carlspring.cloud.storage.s3fs; +import org.carlspring.cloud.storage.s3fs.junit.annotations.MinioIntegrationTest; import org.carlspring.cloud.storage.s3fs.util.EnvironmentBuilder; import org.carlspring.cloud.storage.s3fs.util.MinioContainer; -import static org.carlspring.cloud.storage.s3fs.S3Factory.ACCESS_KEY; -import static org.carlspring.cloud.storage.s3fs.S3Factory.SECRET_KEY; +import java.lang.reflect.Modifier; +import java.net.URI; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.carlspring.cloud.storage.s3fs.S3Factory.*; /** * This abstract class holds common integration test logic. @@ -12,6 +18,8 @@ public abstract class BaseIntegrationTest extends BaseTest { + private static MinioContainer minioContainer; + static { if (isMinioEnv()) @@ -21,8 +29,11 @@ public abstract class BaseIntegrationTest extends BaseTest final String accessKey = (String) EnvironmentBuilder.getRealEnv().get(ACCESS_KEY); final String secretKey = (String) EnvironmentBuilder.getRealEnv().get(SECRET_KEY); final String bucketName = (String) EnvironmentBuilder.getRealEnv().get(EnvironmentBuilder.BUCKET_NAME_KEY); - final MinioContainer minioContainer = new MinioContainer(accessKey, secretKey, bucketName); + minioContainer = new MinioContainer(accessKey, secretKey, bucketName); minioContainer.start(); + + // Minio is using HTTP. + System.setProperty(PROTOCOL, "http"); } catch (Exception e) { @@ -34,7 +45,68 @@ public abstract class BaseIntegrationTest extends BaseTest public static boolean isMinioEnv() { String integrationTestType = System.getProperty("running.it"); - return integrationTestType != null && integrationTestType.equalsIgnoreCase("minio"); + boolean propertyWins = integrationTestType != null && integrationTestType.equalsIgnoreCase("minio"); + + // For when you are running via IDEs. + if(!propertyWins) { + // Filter out noise and leave only `org.carlspring.cloud.storage.s3fs` calls in the stack trace. + List elements = Arrays.stream(Thread.currentThread().getStackTrace()) + .filter(e -> { + String basePackage = BaseTest.class.getPackage().getName(); + String packageName = null; + try + { + packageName = Class.forName(e.getClassName()).getPackage().getName(); + } + catch (ClassNotFoundException classNotFoundException) + { + // This is commented out, because sometimes when running + // in debug mode from an IDE the stacktrace will contain + // gradle classes which cannot be found. + //classNotFoundException.printStackTrace(); + } + return packageName != null && packageName.startsWith(basePackage); + }) + .collect(Collectors.toList()); + + if (elements.size() > 0) + { + StackTraceElement last = elements.get(elements.size() - 1); + String className = last.getClassName(); + try + { + Class clazz = Class.forName(className); + int modifiers = clazz.getModifiers(); + boolean isAbstract = Modifier.isAbstract(modifiers); + + if (!isAbstract) + { + return Arrays.stream(clazz.getDeclaredAnnotations()) + .anyMatch(a -> a.annotationType() == MinioIntegrationTest.class); + } + } + catch (ClassNotFoundException e) + { + e.printStackTrace(); + } + + } + + return false; + + } + + + return propertyWins; + } + + public static MinioContainer getMinioContainer() + { + return minioContainer; + } + + public static URI getS3URIForMinioContainer() { + return EnvironmentBuilder.getS3URI(URI.create("s3://localhost:" + getMinioContainer().getMappedPort(9000) + "/")); } } diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/BaseTest.java b/src/test/java/org/carlspring/cloud/storage/s3fs/BaseTest.java index c62db152..6ba289ca 100644 --- a/src/test/java/org/carlspring/cloud/storage/s3fs/BaseTest.java +++ b/src/test/java/org/carlspring/cloud/storage/s3fs/BaseTest.java @@ -9,6 +9,8 @@ import ch.qos.logback.classic.pattern.TargetLengthBasedClassNameAbbreviator; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; + import static java.util.UUID.randomUUID; /** @@ -43,7 +45,10 @@ protected String getTestBasePath() } catch (ClassNotFoundException classNotFoundException) { - classNotFoundException.printStackTrace(); + // This is commented out, because sometimes when running + // in debug mode from an IDE the stacktrace will contain + // gradle classes which cannot be found. + //classNotFoundException.printStackTrace(); } return packageName != null && packageName.startsWith(basePackage); @@ -63,9 +68,17 @@ protected String getTestBasePath() if (!isAbstract) { - Method method = clazz.getDeclaredMethod(methodName); - Test hasTestAnnotation = method.getDeclaredAnnotation(Test.class); - if (hasTestAnnotation != null) + Method method = Arrays.stream(clazz.getDeclaredMethods()) + .filter(m -> m.getName().equals(methodName)) + .findFirst() + .orElse(null); + if(method == null) { + throw new NoSuchMethodException(className+"#"+methodName); + } + boolean hasTestAnnotation = Arrays.stream(method.getDeclaredAnnotations()) + .anyMatch(a -> a.annotationType() == Test.class || + a.annotationType() == ParameterizedTest.class); + if (hasTestAnnotation) { // Additional prefix after the class name for better differentiation. String prNumber = System.getenv(PR_NUMBER_ENV_VAR); diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/FilesIT.java b/src/test/java/org/carlspring/cloud/storage/s3fs/FilesIT.java index cb2effa1..38debee2 100644 --- a/src/test/java/org/carlspring/cloud/storage/s3fs/FilesIT.java +++ b/src/test/java/org/carlspring/cloud/storage/s3fs/FilesIT.java @@ -12,6 +12,7 @@ import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.util.EnumSet; +import java.util.HashMap; import java.util.Map; import java.util.UUID; @@ -23,6 +24,9 @@ import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; + +import static java.util.UUID.randomUUID; +import static org.assertj.core.api.Assertions.assertThat; import static org.carlspring.cloud.storage.s3fs.util.S3EndpointConstant.S3_GLOBAL_URI_IT; import static org.junit.jupiter.api.Assertions.*; @@ -64,7 +68,9 @@ private static FileSystem build() private static FileSystem createNewFileSystem() throws IOException { - return FileSystems.newFileSystem(uriGlobal, EnvironmentBuilder.getRealEnv()); + Map env = new HashMap<>(EnvironmentBuilder.getRealEnv()); + env.put(S3Factory.REQUEST_HEADER_CACHE_CONTROL, "no-cache"); + return FileSystems.newFileSystem(uriGlobal, env); } @Test @@ -452,6 +458,19 @@ void copyDownload() assertArrayEquals(Files.readAllBytes(result), Files.readAllBytes(notExistLocalResult)); } + @Test + void copyAsNewFileInS3() + throws IOException + { + Path sourceFile = uploadSingleFile(null); + Path targetFile = sourceFile.getParent().resolve("copyAsNewFileInS3-" + UUID.randomUUID()); + + Files.copy(sourceFile, targetFile); + + assertTrue(Files.exists(targetFile)); + assertArrayEquals(Files.readAllBytes(sourceFile), Files.readAllBytes(targetFile)); + } + @Test void moveFromDifferentProviders() throws IOException @@ -739,6 +758,95 @@ void fileIsReadableBucketFile() assertTrue(readable); } + @Test + void shouldReplaceExistingFileExample1() + throws IOException + { + + Path file = Files.createTempFile("file-it-srefe1-1-", "file"); + Path dw1 = Files.createTempFile("file-it-srefe1-dw-1-", "file"); + Path dw2 = Files.createTempFile("file-it-srefe1-dw-2-", "file"); + Path dw3 = Files.createTempFile("file-it-srefe1-dw-3-", "file"); + + Files.write(file, "first".getBytes(), StandardOpenOption.APPEND); + + String filename = randomUUID().toString(); + String key = getTestBasePath() + "/" + filename; + + Path s3file = fileSystemAmazon.getPath(bucket, key); + + String first = "first-write"; + Files.write(s3file, first.getBytes()); + assertThat(Files.readAllBytes(s3file)).isEqualTo(first.getBytes()); + + Files.copy(s3file, dw1, StandardCopyOption.REPLACE_EXISTING); + assertThat(dw1).hasBinaryContent(first.getBytes()); + + String second = "second-write"; + Files.write(s3file, second.getBytes()); + assertThat(Files.readAllBytes(s3file)).isEqualTo(second.getBytes()); + + Files.copy(s3file, dw2, StandardCopyOption.REPLACE_EXISTING); + assertThat(dw2).hasBinaryContent(second.getBytes()); + + String third = "third-write"; + Files.write(s3file, third.getBytes()); + assertThat(Files.readAllBytes(s3file)).isEqualTo(third.getBytes()); + + Files.copy(s3file, dw3, StandardCopyOption.REPLACE_EXISTING); + assertThat(dw3).hasBinaryContent(third.getBytes()); + + } + + @Test + void shouldReplaceExistingFileExample2() + throws IOException + { + + Path file = Files.createTempFile("file-it-srefe2-1-", "file"); + Path dw1 = Files.createTempFile("file-it-srefe2-dw-1-", "file"); + Path dw2 = Files.createTempFile("file-it-srefe2-dw-2-", "file"); + Path dw3 = Files.createTempFile("file-it-srefe2-dw-3-", "file"); + + Files.write(file, "first".getBytes(), StandardOpenOption.APPEND); + + String filename = randomUUID().toString(); + String key = getTestBasePath() + "/" + filename; + + Path s3file = fileSystemAmazon.getPath(bucket, key); + Files.createFile(s3file); + + String first = "first-write"; + + try(OutputStream out = Files.newOutputStream(s3file, StandardOpenOption.TRUNCATE_EXISTING)) { + out.write(first.getBytes()); + } + assertThat(Files.readAllBytes(s3file)).isEqualTo(first.getBytes()); + + Files.copy(s3file, dw1, StandardCopyOption.REPLACE_EXISTING); + assertThat(dw1).hasBinaryContent(first.getBytes()); + + String second = "second-write"; + try(OutputStream out = Files.newOutputStream(s3file, StandardOpenOption.TRUNCATE_EXISTING)) { + out.write(second.getBytes()); + } + assertThat(Files.readAllBytes(s3file)).isEqualTo(second.getBytes()); + + Files.copy(s3file, dw2, StandardCopyOption.REPLACE_EXISTING); + assertThat(dw2).hasBinaryContent(second.getBytes()); + + String third = "third-write"; + try(OutputStream out = Files.newOutputStream(s3file, StandardOpenOption.TRUNCATE_EXISTING)) { + out.write(third.getBytes()); + } + assertThat(Files.readAllBytes(s3file)).isEqualTo(third.getBytes()); + + Files.copy(s3file, dw3, StandardCopyOption.REPLACE_EXISTING); + assertThat(dw3).hasBinaryContent(third.getBytes()); + + } + + // helpers private Path createEmptyDir() diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/S3OutputStreamTest.java b/src/test/java/org/carlspring/cloud/storage/s3fs/S3OutputStreamTest.java index 5d9f05e1..e480a66f 100644 --- a/src/test/java/org/carlspring/cloud/storage/s3fs/S3OutputStreamTest.java +++ b/src/test/java/org/carlspring/cloud/storage/s3fs/S3OutputStreamTest.java @@ -2,21 +2,7 @@ import org.carlspring.cloud.storage.s3fs.util.S3ClientMock; import org.carlspring.cloud.storage.s3fs.util.S3MockFactory; - -import java.io.DataInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.lang.reflect.Method; -import java.util.HashMap; -import java.util.List; -import java.util.Optional; -import java.util.Random; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -28,28 +14,25 @@ import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.http.Header; -import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.*; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.UUID.randomUUID; import static org.carlspring.cloud.storage.s3fs.S3OutputStream.MAX_ALLOWED_UPLOAD_PARTS; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.atLeastOnce; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; @ExtendWith(MockitoExtension.class) -class S3OutputStreamTest +class S3OutputStreamTest extends BaseTest { private static final String BUCKET_NAME = "s3OutputStreamTest"; @@ -60,8 +43,6 @@ class S3OutputStreamTest @Captor private ArgumentCaptor requestBodyCaptor; - private String key; - private static Stream offsetAndLengthForWriteProvider() { return Stream.of( @@ -73,20 +54,13 @@ private static Stream offsetAndLengthForWriteProvider() ); } - @BeforeEach - void init(final TestInfo testInfo) - { - final Optional method = testInfo.getTestMethod(); - key = method.map(Method::getName).orElseThrow(() -> new IllegalStateException("No method name ?")); - } - @Test void openAndCloseProducesEmptyObject() throws IOException { //given + final String key = getTestBasePath() + "/" + randomUUID(); final S3ClientMock client = S3MockFactory.getS3ClientMock(); - final S3ObjectId objectId = S3ObjectId.builder() .bucket(BUCKET_NAME) .key(key) @@ -107,8 +81,8 @@ void zeroBytesWrittenProduceEmptyObject() throws IOException { //given + final String key = getTestBasePath() + "/" + randomUUID(); final S3ClientMock client = S3MockFactory.getS3ClientMock(); - final S3ObjectId objectId = S3ObjectId.builder() .bucket(BUCKET_NAME) .key(key) @@ -131,8 +105,8 @@ void invalidValuesForOffsetAndLengthProducesIndexOutOfBoundsException(final int final int length) { //given + final String key = getTestBasePath() + "/" + randomUUID(); final S3ClientMock client = S3MockFactory.getS3ClientMock(); - final S3ObjectId objectId = S3ObjectId.builder() .bucket(BUCKET_NAME) .key(key) @@ -155,6 +129,7 @@ void closeAndWriteProducesIOException() throws IOException { //given + final String key = getTestBasePath() + "/" + randomUUID(); final S3ClientMock client = S3MockFactory.getS3ClientMock(); client.bucket(BUCKET_NAME).file(key); @@ -187,6 +162,7 @@ void maxNumberOfUploadPartsReachedProducesIOException() throws IOException { //given + final String key = getTestBasePath() + "/" + randomUUID(); final S3ClientMock client = S3MockFactory.getS3ClientMock(); client.bucket(BUCKET_NAME).file(key); @@ -219,6 +195,7 @@ void smallDataUsesPutObject() throws IOException { //given + final String key = getTestBasePath() + "/" + randomUUID(); final S3ClientMock client = S3MockFactory.getS3ClientMock(); client.bucket(BUCKET_NAME).file(key); @@ -243,6 +220,7 @@ void bigDataUsesMultipartUpload() throws IOException { //given + final String key = getTestBasePath() + "/" + randomUUID(); final S3ClientMock client = S3MockFactory.getS3ClientMock(); client.bucket(BUCKET_NAME).file(key); @@ -272,6 +250,7 @@ void whenCreateMultipartUploadFailsThenAnExceptionIsThrown() throws IOException { //given + final String key = getTestBasePath() + "/" + randomUUID(); final S3ClientMock client = S3MockFactory.getS3ClientMock(); client.bucket(BUCKET_NAME).file(key); @@ -311,6 +290,7 @@ void whenCreateMultipartUploadReturnsNullUploadIdThenAnExceptionIsThrown() throws IOException { //given + final String key = getTestBasePath() + "/" + randomUUID(); final S3ClientMock client = S3MockFactory.getS3ClientMock(); client.bucket(BUCKET_NAME).file(key); @@ -352,6 +332,7 @@ void whenUploadPartFailsThenAnExceptionIsThrown() throws IOException { //given + final String key = getTestBasePath() + "/" + randomUUID(); final S3ClientMock client = S3MockFactory.getS3ClientMock(); client.bucket(BUCKET_NAME).file(key); @@ -386,6 +367,7 @@ void whenUploadPartAndAbortMultipartFailsThenAnExceptionIsThrown() throws IOException { //given + final String key = getTestBasePath() + "/" + randomUUID(); final S3ClientMock client = S3MockFactory.getS3ClientMock(); client.bucket(BUCKET_NAME).file(key); diff --git a/src/test/java/org/carlspring/cloud/storage/s3fs/util/EnvironmentBuilder.java b/src/test/java/org/carlspring/cloud/storage/s3fs/util/EnvironmentBuilder.java index efd926ab..c974fd69 100644 --- a/src/test/java/org/carlspring/cloud/storage/s3fs/util/EnvironmentBuilder.java +++ b/src/test/java/org/carlspring/cloud/storage/s3fs/util/EnvironmentBuilder.java @@ -68,11 +68,11 @@ public static Map getRealEnv() throw new RuntimeException("Unable to load properties file from classpath nor to find environment variables!", e); } - env = ImmutableMap.builder().put(ACCESS_KEY, props.getProperty(ACCESS_KEY)) - .put(SECRET_KEY, props.getProperty(SECRET_KEY)) - .put(REGION, props.getProperty(REGION)) - .put(PROTOCOL, props.getProperty(PROTOCOL)) - .put(BUCKET_NAME_KEY, props.getProperty(BUCKET_NAME_KEY)) + env = ImmutableMap.builder().put(ACCESS_KEY, getPropFromSystemOrFallback(ACCESS_KEY, props)) + .put(SECRET_KEY, getPropFromSystemOrFallback(SECRET_KEY, props)) + .put(REGION, getPropFromSystemOrFallback(REGION, props)) + .put(PROTOCOL, getPropFromSystemOrFallback(PROTOCOL, props)) + .put(BUCKET_NAME_KEY, getPropFromSystemOrFallback(BUCKET_NAME_KEY, props)) .build(); } @@ -93,6 +93,10 @@ private static InputStream getPropertiesResource() } } + private static String getPropFromSystemOrFallback(String key, Properties fallback) { + return System.getProperty(key, null) != null ? System.getProperty(key) : fallback.getProperty(key, null); + } + /** * Attempt to retrieve OS Environment Variable * @return