From 614653ac30fca029dbf41ba68d885378b6407881 Mon Sep 17 00:00:00 2001 From: Kartik Ganesh Date: Tue, 12 Dec 2023 12:04:46 -0800 Subject: [PATCH] [POC] Add OSIS management and pipeline roles to Migration Console CDK Signed-off-by: Kartik Ganesh --- .../python/osi_data_migration.py | 5 ++-- .../runOSIHistoricalDataMigration.sh | 20 ++++++++++---- .../service-stacks/migration-console-stack.ts | 27 +++++++++++++++++-- 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/python/osi_data_migration.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/python/osi_data_migration.py index 4681f71ed..f1b21a070 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/python/osi_data_migration.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/python/osi_data_migration.py @@ -45,14 +45,15 @@ config = endpoint_utils.get_supported_endpoint_config(pipeline_config, endpoint_utils.SOURCE_KEY)[1] # Fargate stores the current region in the AWS_REGION env var region: str = os.environ.get("AWS_REGION") + pipeline_role_arn: str = os.environ.get("OSIS_PIPELINE_ROLE_ARN") if "disable_authentication" in config: del config["disable_authentication"] - config["aws"] = {"region": region} + config["aws"] = {"region": region, "sts_role_arn": pipeline_role_arn} config = endpoint_utils.get_supported_endpoint_config(pipeline_config, endpoint_utils.SINK_KEY)[1] if "disable_authentication" in config: del config["disable_authentication"] - config["aws"] = {"region": region} + config["aws"] = {"region": region, "sts_role_arn": pipeline_role_arn} # Write OSI pipeline config to output file with open(namespace.output, 'w') as out_file: diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runOSIHistoricalDataMigration.sh b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runOSIHistoricalDataMigration.sh index 2b356d693..1b85d925d 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runOSIHistoricalDataMigration.sh +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runOSIHistoricalDataMigration.sh @@ -7,12 +7,20 @@ fi if [ -e /tmp/temp_pipeline ]; then rm /tmp/temp_pipeline fi +if [ -e /tmp/osi_pipeline.yaml ]; then + rm /tmp/osi_pipeline.yaml +fi # Ensure target cluster endpoint is available as an env var if [ -z "$MIGRATION_DOMAIN_ENDPOINT" ]; then echo "MIGRATION_DOMAIN_ENDPOINT environment variable not found for target cluster endpoint, exiting..." exit 1 fi +# Ensure OSIS pipeline role ARN is available as an env var +if [ -z "$OSIS_PIPELINE_ROLE_ARN" ]; then + echo "OSIS_PIPELINE_ROLE_ARN environment variable not found for OSIS pipeline role, exiting..." + exit 1 +fi # Default values secret_name="dev-default-fetch-migration-pipelineConfig" @@ -34,8 +42,10 @@ done # Get pipeline config from secrets manager pipeline_config=`aws secretsmanager get-secret-value --secret-id $secret_name | jq -r '.SecretString' | base64 -d` +# Remove any port from target endpoint because OSIS doesn't allow it +target_endpoint=${MIGRATION_DOMAIN_ENDPOINT%:[0-9]*} # Replace target cluster placeholder with actual endpoint value -pipeline_config=${pipeline_config//$MIGRATION_DOMAIN_ENDPOINT} +pipeline_config=${pipeline_config//$target_endpoint} # Write output to temp file for use by metadata migration cat <<<$pipeline_config > /tmp/user_pipeline # Setup and run metadata migration @@ -46,13 +56,13 @@ python3 metadata_migration.py -r /tmp/user_pipeline /tmp/temp_pipeline python3 osi_data_migration.py /tmp/temp_pipeline /tmp/osi_pipeline.yaml cd .. cat /tmp/osi_pipeline.yaml -# TODO Add role config to OSI pipeline before we can create the pipeline from here -#aws osis create-pipeline --pipeline-name fetch-migration --min-units 1 --max-units 1 --pipeline-configuration-body /tmp/osi_pipeline.yaml +# TODO Need to wire in VPC config +#aws osis create-pipeline --pipeline-name osi-fetch-migration --min-units 1 --max-units 1 --pipeline-configuration-body file:///tmp/osi_pipeline.yaml --log-publishing-options IsLoggingEnabled=true,CloudWatchLogDestination={LogGroup=/aws/vendedlogs/OpenSearchService/pipelines/osi-fetch-migration} --vpc-options SubnetIds=s1,s2,SecurityGroupIds=sg1 # Clean up state if [ -e /tmp/user_pipeline ]; then rm /tmp/user_pipeline fi -if [ -e /tmp/osi_pipeline.yaml ]; then - rm /tmp/osi_pipeline.yaml +if [ -e /tmp/temp_pipeline.yaml ]; then + rm /tmp/temp_pipeline.yaml fi diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts index 2dfee784d..58f2d4ee0 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts @@ -5,7 +5,7 @@ import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; import {StringParameter} from "aws-cdk-lib/aws-ssm"; -import {Effect, PolicyStatement} from "aws-cdk-lib/aws-iam"; +import {Effect, PolicyStatement, Role, ServicePrincipal} from "aws-cdk-lib/aws-iam"; import {createOpenSearchIAMAccessPolicy, createOpenSearchServerlessIAMAccessPolicy} from "../common-utilities"; @@ -99,6 +99,18 @@ export class MigrationConsoleStack extends MigrationServiceCore { if (props.fetchMigrationEnabled) { environment["FETCH_MIGRATION_COMMAND"] = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/fetchMigrationCommand`) + // [POC] Add a pipeline role for OSIS + const osisPipelineRole = new Role(this, 'osisPipelineRole', { + assumedBy: new ServicePrincipal('osis-pipelines.amazonaws.com'), + description: 'OSIS Pipeline role for Fetch Migration' + }); + // Add policy to allow access to Opensearch domains + osisPipelineRole.addToPolicy(new PolicyStatement({ + effect: Effect.ALLOW, + actions: ["es:DescribeDomain", "es:ESHttp*"], + resources: [`arn:aws:es:${props.env?.region}:${props.env?.account}:domain/*`] + })) + environment["OSIS_PIPELINE_ROLE_ARN"] = osisPipelineRole.roleArn const fetchMigrationTaskDefArn = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/fetchMigrationTaskDefArn`); const fetchMigrationTaskRunPolicy = new PolicyStatement({ @@ -111,9 +123,10 @@ export class MigrationConsoleStack extends MigrationServiceCore { const fetchMigrationTaskRoleArn = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/fetchMigrationTaskRoleArn`); const fetchMigrationTaskExecRoleArn = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/fetchMigrationTaskExecRoleArn`); // Required as per https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-iam-roles.html + // [POC] Allow passing of pipeline role const fetchMigrationPassRolePolicy = new PolicyStatement({ effect: Effect.ALLOW, - resources: [fetchMigrationTaskRoleArn, fetchMigrationTaskExecRoleArn], + resources: [fetchMigrationTaskRoleArn, fetchMigrationTaskExecRoleArn, osisPipelineRole.roleArn], actions: [ "iam:PassRole" ] @@ -126,9 +139,19 @@ export class MigrationConsoleStack extends MigrationServiceCore { "secretsmanager:GetSecretValue" ] }) + + // [POC] Enable OSIS management from Migration Console + const osisManagementPolicy = new PolicyStatement({ + effect: Effect.ALLOW, + resources: ["*"], + actions: [ + "osis:*" + ] + }) taskRolePolicies.push(fetchMigrationTaskRunPolicy) taskRolePolicies.push(fetchMigrationPassRolePolicy) taskRolePolicies.push(osiMigrationGetSecretPolicy) + taskRolePolicies.push(osisManagementPolicy) } this.createService({