diff --git a/.github/workflows/push_pr.yml b/.github/workflows/push_pr.yml index 82da4df3..6129ea18 100644 --- a/.github/workflows/push_pr.yml +++ b/.github/workflows/push_pr.yml @@ -23,7 +23,7 @@ concurrency: cancel-in-progress: true jobs: compile_and_test: - uses: ./.github/workflows/common.yml + uses: apache/flink-connector-shared-utils/.github/workflows/ci.yml@ci_utils strategy: matrix: flink: [1.17.1, 1.18.0] @@ -31,4 +31,6 @@ jobs: flink_version: ${{ matrix.flink }} flink_url: https://archive.apache.org/dist/flink/flink-${{ matrix.flink }}/flink-${{ matrix.flink }}-bin-scala_2.12.tgz cache_flink_binary: true - secrets: inherit + env: + FLINK_AWS_USER: ${{ secrets.FLINK_AWS_USER }} + FLINK_AWS_PASSWORD: ${{ secrets.FLINK_AWS_PASSWORD }} \ No newline at end of file diff --git a/flink-connector-aws-e2e-tests/flink-connector-aws-kinesis-streams-e2e-tests/src/test/java/org/apache/flink/connector/kinesis/sink/KinesisStreamsSinkITCase.java b/flink-connector-aws-e2e-tests/flink-connector-aws-kinesis-streams-e2e-tests/src/test/java/org/apache/flink/connector/kinesis/sink/KinesisStreamsSinkITCase.java index 9287d2e1..d97d342d 100644 --- a/flink-connector-aws-e2e-tests/flink-connector-aws-kinesis-streams-e2e-tests/src/test/java/org/apache/flink/connector/kinesis/sink/KinesisStreamsSinkITCase.java +++ b/flink-connector-aws-e2e-tests/flink-connector-aws-kinesis-streams-e2e-tests/src/test/java/org/apache/flink/connector/kinesis/sink/KinesisStreamsSinkITCase.java @@ -23,6 +23,7 @@ import org.apache.flink.connector.aws.util.AWSGeneralUtil; import org.apache.flink.connector.kinesis.testutils.AWSEndToEndTestUtils; import org.apache.flink.connector.kinesis.testutils.AWSKinesisResourceManager; +import org.apache.flink.connector.kinesis.testutils.RequiresAwsCredentials; import org.apache.flink.runtime.client.JobExecutionException; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -33,8 +34,6 @@ import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.parallel.Execution; @@ -52,8 +51,8 @@ /** End-to-end tests for Kinesis Data Streams Sink. */ @ExtendWith(MiniClusterExtension.class) @Execution(ExecutionMode.CONCURRENT) -@Tag("requires-aws-credentials") -@Disabled +@RequiresAwsCredentials +// @Disabled class KinesisStreamsSinkITCase { private static final String DEFAULT_FIRST_SHARD_NAME = "shardId-000000000000"; diff --git a/flink-connector-aws-e2e-tests/flink-connector-aws-kinesis-streams-e2e-tests/src/test/java/org/apache/flink/connector/kinesis/testutils/AWSEndToEndTestUtils.java b/flink-connector-aws-e2e-tests/flink-connector-aws-kinesis-streams-e2e-tests/src/test/java/org/apache/flink/connector/kinesis/testutils/AWSEndToEndTestUtils.java index a8d2824a..2fcf87c6 100644 --- a/flink-connector-aws-e2e-tests/flink-connector-aws-kinesis-streams-e2e-tests/src/test/java/org/apache/flink/connector/kinesis/testutils/AWSEndToEndTestUtils.java +++ b/flink-connector-aws-e2e-tests/flink-connector-aws-kinesis-streams-e2e-tests/src/test/java/org/apache/flink/connector/kinesis/testutils/AWSEndToEndTestUtils.java @@ -22,6 +22,7 @@ import org.apache.flink.connector.aws.util.AWSGeneralUtil; import org.apache.flink.util.Preconditions; +import com.google.common.base.Strings; import software.amazon.awssdk.awscore.client.builder.AwsClientBuilder; import software.amazon.awssdk.awscore.client.builder.AwsSyncClientBuilder; import software.amazon.awssdk.core.SdkClient; @@ -62,11 +63,11 @@ public static Properties createTestConfig() { } private static void configureTestCredentials(Properties config) { - Preconditions.checkNotNull( - System.getenv("FLINK_AWS_USER"), + Preconditions.checkArgument( + !Strings.isNullOrEmpty(System.getenv("FLINK_AWS_USER")), "FLINK_AWS_USER not configured for end to end test."); - Preconditions.checkNotNull( - System.getenv("FLINK_AWS_PASSWORD"), + Preconditions.checkArgument( + !Strings.isNullOrEmpty(System.getenv("FLINK_AWS_PASSWORD")), "FLINK_AWS_PASSWORD not configured for end to end test."); config.setProperty( AWSConfigConstants.accessKeyId(AWS_CREDENTIALS_PROVIDER), diff --git a/flink-connector-aws-e2e-tests/flink-connector-aws-kinesis-streams-e2e-tests/src/test/java/org/apache/flink/connector/kinesis/testutils/RequiresAwsCredentials.java b/flink-connector-aws-e2e-tests/flink-connector-aws-kinesis-streams-e2e-tests/src/test/java/org/apache/flink/connector/kinesis/testutils/RequiresAwsCredentials.java new file mode 100644 index 00000000..757a4ae9 --- /dev/null +++ b/flink-connector-aws-e2e-tests/flink-connector-aws-kinesis-streams-e2e-tests/src/test/java/org/apache/flink/connector/kinesis/testutils/RequiresAwsCredentials.java @@ -0,0 +1,26 @@ +package org.apache.flink.connector.kinesis.testutils; + +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * The {@code RequiresAwsCredentials} annotation is used to indicate that a method or a class + * requires AWS credentials to be provided as environment variables. This annotation can be applied + * to both methods and classes. + * + *

Note that in order to use this annotation, the following environment variables must be set: + * + *

+ */ +@Target({ElementType.METHOD, ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +@EnabledIfEnvironmentVariable(named = "FLINK_AWS_USER", matches = ".+") +@EnabledIfEnvironmentVariable(named = "FLINK_AWS_PASSWORD", matches = ".+") +public @interface RequiresAwsCredentials {} diff --git a/flink-connector-aws-e2e-tests/flink-connector-aws-kinesis-streams-e2e-tests/src/test/java/org/apache/flink/connector/kinesis/testutils/StaleResourceCleanupITCase.java b/flink-connector-aws-e2e-tests/flink-connector-aws-kinesis-streams-e2e-tests/src/test/java/org/apache/flink/connector/kinesis/testutils/StaleResourceCleanupITCase.java index 66cad7c0..e2dc6f5a 100644 --- a/flink-connector-aws-e2e-tests/flink-connector-aws-kinesis-streams-e2e-tests/src/test/java/org/apache/flink/connector/kinesis/testutils/StaleResourceCleanupITCase.java +++ b/flink-connector-aws-e2e-tests/flink-connector-aws-kinesis-streams-e2e-tests/src/test/java/org/apache/flink/connector/kinesis/testutils/StaleResourceCleanupITCase.java @@ -20,13 +20,12 @@ import org.apache.flink.connector.aws.testutils.AWSServicesTestUtils; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import software.amazon.awssdk.http.SdkHttpClient; import software.amazon.awssdk.services.kinesis.KinesisClient; /** Dummy test to clean up stale resources. */ -@Tag("requires-aws-credentials") +@RequiresAwsCredentials public class StaleResourceCleanupITCase { @Test void cleanUpStaleKinesisStreams() {