From 7ab7ef8e199bc7fbee6c6b8f250865c86b4f3314 Mon Sep 17 00:00:00 2001 From: Kartik Ganesh Date: Tue, 12 Dec 2023 12:04:46 -0800 Subject: [PATCH] [POC] Add OSIS management and pipeline roles to Migration Console CDK Signed-off-by: Kartik Ganesh --- .../python/metadata_migration.py | 2 ++ .../python/osi_data_migration.py | 7 +++-- .../runOSIHistoricalDataMigration.sh | 23 +++++++++++----- .../service-stacks/migration-console-stack.ts | 27 +++++++++++++++++-- 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/python/metadata_migration.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/python/metadata_migration.py index aaff51cc9..0890a606d 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/python/metadata_migration.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/python/metadata_migration.py @@ -108,6 +108,8 @@ def run(args: MetadataMigrationParams) -> MetadataMigrationResult: if __name__ == '__main__': # pragma no cover + # Set log level + logging.basicConfig(level=logging.INFO) # Set up parsing for command line arguments arg_parser = argparse.ArgumentParser( prog="python metadata_migration.py", diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/python/osi_data_migration.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/python/osi_data_migration.py index 4681f71ed..65bfbc199 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/python/osi_data_migration.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/python/osi_data_migration.py @@ -13,6 +13,8 @@ import endpoint_utils if __name__ == '__main__': + # Set log level + logging.basicConfig(level=logging.INFO) # Set up parsing for command line arguments arg_parser = argparse.ArgumentParser( prog="python osi_data_migration.py", @@ -45,14 +47,15 @@ config = endpoint_utils.get_supported_endpoint_config(pipeline_config, endpoint_utils.SOURCE_KEY)[1] # Fargate stores the current region in the AWS_REGION env var region: str = os.environ.get("AWS_REGION") + pipeline_role_arn: str = os.environ.get("OSIS_PIPELINE_ROLE_ARN") if "disable_authentication" in config: del config["disable_authentication"] - config["aws"] = {"region": region} + config["aws"] = {"region": region, "sts_role_arn": pipeline_role_arn} config = endpoint_utils.get_supported_endpoint_config(pipeline_config, endpoint_utils.SINK_KEY)[1] if "disable_authentication" in config: del config["disable_authentication"] - config["aws"] = {"region": region} + config["aws"] = {"region": region, "sts_role_arn": pipeline_role_arn} # Write OSI pipeline config to output file with open(namespace.output, 'w') as out_file: diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runOSIHistoricalDataMigration.sh b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runOSIHistoricalDataMigration.sh index 2b356d693..296927b12 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runOSIHistoricalDataMigration.sh +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runOSIHistoricalDataMigration.sh @@ -7,12 +7,20 @@ fi if [ -e /tmp/temp_pipeline ]; then rm /tmp/temp_pipeline fi +if [ -e /tmp/osi_pipeline.yaml ]; then + rm /tmp/osi_pipeline.yaml +fi # Ensure target cluster endpoint is available as an env var if [ -z "$MIGRATION_DOMAIN_ENDPOINT" ]; then echo "MIGRATION_DOMAIN_ENDPOINT environment variable not found for target cluster endpoint, exiting..." exit 1 fi +# Ensure OSIS pipeline role ARN is available as an env var +if [ -z "$OSIS_PIPELINE_ROLE_ARN" ]; then + echo "OSIS_PIPELINE_ROLE_ARN environment variable not found for OSIS pipeline role, exiting..." + exit 1 +fi # Default values secret_name="dev-default-fetch-migration-pipelineConfig" @@ -34,25 +42,28 @@ done # Get pipeline config from secrets manager pipeline_config=`aws secretsmanager get-secret-value --secret-id $secret_name | jq -r '.SecretString' | base64 -d` +# Remove any port from target endpoint because OSIS doesn't allow it +target_endpoint=${MIGRATION_DOMAIN_ENDPOINT%:[0-9]*} # Replace target cluster placeholder with actual endpoint value -pipeline_config=${pipeline_config//$MIGRATION_DOMAIN_ENDPOINT} +pipeline_config=${pipeline_config//$target_endpoint} # Write output to temp file for use by metadata migration cat <<<$pipeline_config > /tmp/user_pipeline # Setup and run metadata migration cd python/ pip3 install --user -r requirements.txt -# TODO - Configure Python logs to output to console +# Run metadata migration python3 metadata_migration.py -r /tmp/user_pipeline /tmp/temp_pipeline +# Parse output file from previous step to create OSI pipeline input python3 osi_data_migration.py /tmp/temp_pipeline /tmp/osi_pipeline.yaml cd .. cat /tmp/osi_pipeline.yaml -# TODO Add role config to OSI pipeline before we can create the pipeline from here -#aws osis create-pipeline --pipeline-name fetch-migration --min-units 1 --max-units 1 --pipeline-configuration-body /tmp/osi_pipeline.yaml +# TODO Need to wire in VPC config +#aws osis create-pipeline --pipeline-name osi-fetch-migration --min-units 1 --max-units 1 --pipeline-configuration-body file:///tmp/osi_pipeline.yaml --log-publishing-options IsLoggingEnabled=true,CloudWatchLogDestination={LogGroup=/aws/vendedlogs/OpenSearchService/pipelines/osi-fetch-migration} --vpc-options SubnetIds=s1,s2,SecurityGroupIds=sg1 # Clean up state if [ -e /tmp/user_pipeline ]; then rm /tmp/user_pipeline fi -if [ -e /tmp/osi_pipeline.yaml ]; then - rm /tmp/osi_pipeline.yaml +if [ -e /tmp/temp_pipeline.yaml ]; then + rm /tmp/temp_pipeline.yaml fi diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts index 2dfee784d..58f2d4ee0 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts @@ -5,7 +5,7 @@ import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; import {StringParameter} from "aws-cdk-lib/aws-ssm"; -import {Effect, PolicyStatement} from "aws-cdk-lib/aws-iam"; +import {Effect, PolicyStatement, Role, ServicePrincipal} from "aws-cdk-lib/aws-iam"; import {createOpenSearchIAMAccessPolicy, createOpenSearchServerlessIAMAccessPolicy} from "../common-utilities"; @@ -99,6 +99,18 @@ export class MigrationConsoleStack extends MigrationServiceCore { if (props.fetchMigrationEnabled) { environment["FETCH_MIGRATION_COMMAND"] = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/fetchMigrationCommand`) + // [POC] Add a pipeline role for OSIS + const osisPipelineRole = new Role(this, 'osisPipelineRole', { + assumedBy: new ServicePrincipal('osis-pipelines.amazonaws.com'), + description: 'OSIS Pipeline role for Fetch Migration' + }); + // Add policy to allow access to Opensearch domains + osisPipelineRole.addToPolicy(new PolicyStatement({ + effect: Effect.ALLOW, + actions: ["es:DescribeDomain", "es:ESHttp*"], + resources: [`arn:aws:es:${props.env?.region}:${props.env?.account}:domain/*`] + })) + environment["OSIS_PIPELINE_ROLE_ARN"] = osisPipelineRole.roleArn const fetchMigrationTaskDefArn = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/fetchMigrationTaskDefArn`); const fetchMigrationTaskRunPolicy = new PolicyStatement({ @@ -111,9 +123,10 @@ export class MigrationConsoleStack extends MigrationServiceCore { const fetchMigrationTaskRoleArn = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/fetchMigrationTaskRoleArn`); const fetchMigrationTaskExecRoleArn = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/fetchMigrationTaskExecRoleArn`); // Required as per https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-iam-roles.html + // [POC] Allow passing of pipeline role const fetchMigrationPassRolePolicy = new PolicyStatement({ effect: Effect.ALLOW, - resources: [fetchMigrationTaskRoleArn, fetchMigrationTaskExecRoleArn], + resources: [fetchMigrationTaskRoleArn, fetchMigrationTaskExecRoleArn, osisPipelineRole.roleArn], actions: [ "iam:PassRole" ] @@ -126,9 +139,19 @@ export class MigrationConsoleStack extends MigrationServiceCore { "secretsmanager:GetSecretValue" ] }) + + // [POC] Enable OSIS management from Migration Console + const osisManagementPolicy = new PolicyStatement({ + effect: Effect.ALLOW, + resources: ["*"], + actions: [ + "osis:*" + ] + }) taskRolePolicies.push(fetchMigrationTaskRunPolicy) taskRolePolicies.push(fetchMigrationPassRolePolicy) taskRolePolicies.push(osiMigrationGetSecretPolicy) + taskRolePolicies.push(osisManagementPolicy) } this.createService({