[POC] Add OSIS management and pipeline roles to Migration Console CDK

Signed-off-by: Kartik Ganesh <[email protected]>

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/python/metadata_migration.py

@@ -108,6 +108,8 @@ def run(args: MetadataMigrationParams) -> MetadataMigrationResult:
 
 
 if __name__ == '__main__':  # pragma no cover
+    # Set log level
+    logging.basicConfig(level=logging.INFO)
     # Set up parsing for command line arguments
     arg_parser = argparse.ArgumentParser(
         prog="python metadata_migration.py",

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/python/osi_data_migration.py

@@ -13,6 +13,8 @@
 import endpoint_utils
 
 if __name__ == '__main__':
+    # Set log level
+    logging.basicConfig(level=logging.INFO)
     # Set up parsing for command line arguments
     arg_parser = argparse.ArgumentParser(
         prog="python osi_data_migration.py",
@@ -45,14 +47,15 @@
     config = endpoint_utils.get_supported_endpoint_config(pipeline_config, endpoint_utils.SOURCE_KEY)[1]
     # Fargate stores the current region in the AWS_REGION env var
     region: str = os.environ.get("AWS_REGION")
+    pipeline_role_arn: str = os.environ.get("OSIS_PIPELINE_ROLE_ARN")
     if "disable_authentication" in config:
         del config["disable_authentication"]
-    config["aws"] = {"region": region}
+    config["aws"] = {"region": region, "sts_role_arn": pipeline_role_arn}
 
     config = endpoint_utils.get_supported_endpoint_config(pipeline_config, endpoint_utils.SINK_KEY)[1]
     if "disable_authentication" in config:
         del config["disable_authentication"]
-    config["aws"] = {"region": region}
+    config["aws"] = {"region": region, "sts_role_arn": pipeline_role_arn}
 
     # Write OSI pipeline config to output file
     with open(namespace.output, 'w') as out_file:

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/runOSIHistoricalDataMigration.sh

@@ -7,12 +7,20 @@ fi
 if [ -e /tmp/temp_pipeline ]; then
     rm /tmp/temp_pipeline
 fi
+if [ -e /tmp/osi_pipeline.yaml ]; then
+    rm /tmp/osi_pipeline.yaml
+fi
 
 # Ensure target cluster endpoint is available as an env var
 if [ -z "$MIGRATION_DOMAIN_ENDPOINT" ]; then
     echo "MIGRATION_DOMAIN_ENDPOINT environment variable not found for target cluster endpoint, exiting..."
     exit 1
 fi
+# Ensure OSIS pipeline role ARN is available as an env var
+if [ -z "$OSIS_PIPELINE_ROLE_ARN" ]; then
+    echo "OSIS_PIPELINE_ROLE_ARN environment variable not found for OSIS pipeline role, exiting..."
+    exit 1
+fi
 
 # Default values
 secret_name="dev-default-fetch-migration-pipelineConfig"
@@ -34,25 +42,28 @@ done
 
 # Get pipeline config from secrets manager
 pipeline_config=`aws secretsmanager get-secret-value --secret-id $secret_name | jq -r '.SecretString' | base64 -d`
+# Remove any port from target endpoint because OSIS doesn't allow it
+target_endpoint=${MIGRATION_DOMAIN_ENDPOINT%:[0-9]*}
 # Replace target cluster placeholder with actual endpoint value
-pipeline_config=${pipeline_config/<TARGET_CLUSTER_ENDPOINT_PLACEHOLDER>/$MIGRATION_DOMAIN_ENDPOINT}
+pipeline_config=${pipeline_config/<TARGET_CLUSTER_ENDPOINT_PLACEHOLDER>/$target_endpoint}
 # Write output to temp file for use by metadata migration
 cat <<<$pipeline_config > /tmp/user_pipeline
 # Setup and run metadata migration
 cd python/
 pip3 install --user -r requirements.txt
-# TODO - Configure Python logs to output to console
+# Run metadata migration
 python3 metadata_migration.py -r /tmp/user_pipeline /tmp/temp_pipeline
+# Parse output file from previous step to create OSI pipeline input
 python3 osi_data_migration.py /tmp/temp_pipeline /tmp/osi_pipeline.yaml
 cd ..
 cat /tmp/osi_pipeline.yaml
-# TODO Add role config to OSI pipeline before we can create the pipeline from here
-#aws osis create-pipeline --pipeline-name fetch-migration --min-units 1 --max-units 1 --pipeline-configuration-body /tmp/osi_pipeline.yaml
+# TODO Need to wire in VPC config
+#aws osis create-pipeline --pipeline-name osi-fetch-migration --min-units 1 --max-units 1 --pipeline-configuration-body file:///tmp/osi_pipeline.yaml --log-publishing-options IsLoggingEnabled=true,CloudWatchLogDestination={LogGroup=/aws/vendedlogs/OpenSearchService/pipelines/osi-fetch-migration} --vpc-options SubnetIds=s1,s2,SecurityGroupIds=sg1
 
 # Clean up state
 if [ -e /tmp/user_pipeline ]; then
     rm /tmp/user_pipeline
 fi
-if [ -e /tmp/osi_pipeline.yaml ]; then
-    rm /tmp/osi_pipeline.yaml
+if [ -e /tmp/temp_pipeline.yaml ]; then
+    rm /tmp/temp_pipeline.yaml
 fi

deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts

@@ -5,7 +5,7 @@ import {Construct} from "constructs";
 import {join} from "path";
 import {MigrationServiceCore} from "./migration-service-core";
 import {StringParameter} from "aws-cdk-lib/aws-ssm";
-import {Effect, PolicyStatement} from "aws-cdk-lib/aws-iam";
+import {Effect, PolicyStatement, Role, ServicePrincipal} from "aws-cdk-lib/aws-iam";
 import {createOpenSearchIAMAccessPolicy, createOpenSearchServerlessIAMAccessPolicy} from "../common-utilities";
 
 
@@ -99,6 +99,18 @@ export class MigrationConsoleStack extends MigrationServiceCore {
 
         if (props.fetchMigrationEnabled) {
             environment["FETCH_MIGRATION_COMMAND"] = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/fetchMigrationCommand`)
+            // [POC] Add a pipeline role for OSIS
+            const osisPipelineRole = new Role(this, 'osisPipelineRole', {
+                assumedBy: new ServicePrincipal('osis-pipelines.amazonaws.com'),
+                description: 'OSIS Pipeline role for Fetch Migration'
+            });
+            // Add policy to allow access to Opensearch domains
+            osisPipelineRole.addToPolicy(new PolicyStatement({
+                effect: Effect.ALLOW,
+                actions: ["es:DescribeDomain", "es:ESHttp*"],
+                resources: [`arn:aws:es:${props.env?.region}:${props.env?.account}:domain/*`]
+            }))
+            environment["OSIS_PIPELINE_ROLE_ARN"] = osisPipelineRole.roleArn
 
             const fetchMigrationTaskDefArn = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/fetchMigrationTaskDefArn`);
             const fetchMigrationTaskRunPolicy = new PolicyStatement({
@@ -111,9 +123,10 @@ export class MigrationConsoleStack extends MigrationServiceCore {
             const fetchMigrationTaskRoleArn = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/fetchMigrationTaskRoleArn`);
             const fetchMigrationTaskExecRoleArn = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/fetchMigrationTaskExecRoleArn`);
             // Required as per https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-iam-roles.html
+            // [POC] Allow passing of pipeline role
             const fetchMigrationPassRolePolicy = new PolicyStatement({
                 effect: Effect.ALLOW,
-                resources: [fetchMigrationTaskRoleArn, fetchMigrationTaskExecRoleArn],
+                resources: [fetchMigrationTaskRoleArn, fetchMigrationTaskExecRoleArn, osisPipelineRole.roleArn],
                 actions: [
                     "iam:PassRole"
                 ]
@@ -126,9 +139,19 @@ export class MigrationConsoleStack extends MigrationServiceCore {
                     "secretsmanager:GetSecretValue"
                 ]
             })
+
+            // [POC] Enable OSIS management from Migration Console
+            const osisManagementPolicy = new PolicyStatement({
+                effect: Effect.ALLOW,
+                resources: ["*"],
+                actions: [
+                    "osis:*"
+                ]
+            })
             taskRolePolicies.push(fetchMigrationTaskRunPolicy)
             taskRolePolicies.push(fetchMigrationPassRolePolicy)
             taskRolePolicies.push(osiMigrationGetSecretPolicy)
+            taskRolePolicies.push(osisManagementPolicy)
         }
 
         this.createService({