diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/catIndices.sh b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/catIndices.sh index 602efa379..351dc793b 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/catIndices.sh +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/catIndices.sh @@ -64,10 +64,12 @@ if [ "$target_no_auth" = true ]; then fi echo "SOURCE CLUSTER" +curl $source_endpoint/_refresh --insecure $source_auth_string &> /dev/null echo "curl $source_endpoint/_cat/indices?v" curl $source_endpoint/_cat/indices?v --insecure $source_auth_string echo "" echo "TARGET CLUSTER" +curl $target_endpoint/_refresh --insecure $target_auth_string &> /dev/null echo "curl $target_endpoint/_cat/indices?v" curl $target_endpoint/_cat/indices?v --insecure $target_auth_string echo "" \ No newline at end of file diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/setupIntegTests.sh b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/setupIntegTests.sh index c365922fb..75c3caa0b 100755 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/setupIntegTests.sh +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/setupIntegTests.sh @@ -1,9 +1,12 @@ #!/bin/bash -# Example usage: ./setupIntegTests.sh https://github.com/opensearch-project/opensearch-migrations.git main +# Example usage: ./setupIntegTests.sh https://github.com/opensearch-project/opensearch-migrations.git main http://opense-clust-test.elb.us-east-1.amazonaws.com:19200 http://opense-clust-test.elb.us-east-1.amazonaws.com:9200 integ_min_1705067071260_127 git_http_url=$1 branch=$2 +source_endpoint=$3 +proxy_endpoint=$4 +unique_id=$5 mkdir -p /root/integ-tests cd /root/integ-tests || exit @@ -43,6 +46,8 @@ fi source .venv/bin/activate pip install -r requirements.txt echo "Starting python 'tests.py'" -# TODO command to be updated as 'tests.py' gets updated to allow AWS testing -pytest tests.py +# TODO Add support to dynamically skip certain tests depending on setup. Currently, forcing negative auth tests not to run here as the source/target cluster has no auth +set -o xtrace +pytest tests.py::E2ETests::test_0001_index tests.py::E2ETests::test_0002_document tests.py::E2ETests::test_0005_invalidIncorrectUri tests.py::E2ETests::test_0006_OSB --proxy_endpoint="${proxy_endpoint}" --source_endpoint="${source_endpoint}" --target_endpoint="${MIGRATION_DOMAIN_ENDPOINT}" --source_auth_type="none" --target_auth_type="none" --source_verify_ssl=False --target_verify_ssl=False --deployment_type="cloud" --unique_id="${unique_id}" --verbose --junitxml="./reports/${unique_id}.xml" +set +o xtrace deactivate \ No newline at end of file diff --git a/test/awsE2ESolutionSetup.sh b/test/awsE2ESolutionSetup.sh index 9d08a6ded..11fd2bbb1 100755 --- a/test/awsE2ESolutionSetup.sh +++ b/test/awsE2ESolutionSetup.sh @@ -32,9 +32,18 @@ prepare_source_nodes_for_capture () { printf -v group_ids_string '%s ' "${group_ids[@]}" aws ec2 modify-instance-attribute --instance-id $id --groups $group_ids_string fi - echo "Executing ./startCaptureProxy.sh --kafka-endpoints $kafka_brokers on node: $id. Attempting to append output of command: " - command_id=$(aws ssm send-command --instance-ids "$id" --document-name "AWS-RunShellScript" --parameters commands="cd /home/ec2-user/capture-proxy && ./startCaptureProxy.sh --kafka-endpoints $kafka_brokers" --output text --query 'Command.CommandId') - sleep 5 + echo "Executing ./startCaptureProxy.sh --kafka-endpoints $kafka_brokers --git-url $MIGRATIONS_GIT_URL --git-branch $MIGRATIONS_GIT_BRANCH on node: $id" + command_id=$(aws ssm send-command --instance-ids "$id" --document-name "AWS-RunShellScript" --parameters commands="cd /home/ec2-user/capture-proxy && ./startCaptureProxy.sh --kafka-endpoints $kafka_brokers --git-url $MIGRATIONS_GIT_URL --git-branch $MIGRATIONS_GIT_BRANCH" --output text --query 'Command.CommandId') + sleep 10 + command_status=$(aws ssm get-command-invocation --command-id "$command_id" --instance-id "$id" --output text --query 'Status') + # TODO for multi-node setups, we should collect all command ids and allow to run in parallel + while [ "$command_status" != "Success" ] && [ "$command_status" != "Failed" ] + do + echo "Waiting for command to complete, current status is $command_status" + sleep 60 + command_status=$(aws ssm get-command-invocation --command-id "$command_id" --instance-id "$id" --output text --query 'Status') + done + echo "Command has completed with status: $command_status, appending output" echo "Standard Output:" aws ssm get-command-invocation --command-id "$command_id" --instance-id "$id" --output text --query 'StandardOutputContent' echo "Standard Error:" @@ -42,6 +51,21 @@ prepare_source_nodes_for_capture () { done } +restore_and_record () { + deploy_stage=$1 + source_lb_endpoint=$(aws cloudformation describe-stacks --stack-name opensearch-infra-stack-Migration-Source --query "Stacks[0].Outputs[?OutputKey==\`loadbalancerurl\`].OutputValue" --output text) + source_endpoint="http://${source_lb_endpoint}:19200" + kafka_brokers=$(aws ssm get-parameter --name "/migration/$deploy_stage/default/kafkaBrokers" --query 'Parameter.Value' --output text) + console_task_arn=$(aws ecs list-tasks --cluster migration-${deploy_stage}-ecs-cluster --family "migration-${deploy_stage}-migration-console" | jq --raw-output '.taskArns[0]') + + # Print final doc counts and Kafka topic status + unbuffer aws ecs execute-command --cluster "migration-${STAGE}-ecs-cluster" --task "${console_task_arn}" --container "migration-console" --interactive --command "./catIndices.sh --source_endpoint $source_endpoint --source_no_auth --target_no_auth" + unbuffer aws ecs execute-command --cluster "migration-${STAGE}-ecs-cluster" --task "${console_task_arn}" --container "migration-console" --interactive --command "./kafka-tools/kafka/bin/kafka-consumer-groups.sh --bootstrap-server ${kafka_brokers} --timeout 100000 --describe --group logging-group-default --command-config kafka-tools/aws/msk-iam-auth.properties" + + # Turn off Replayer + aws ecs update-service --cluster "migration-${deploy_stage}-ecs-cluster" --service "migration-${deploy_stage}-traffic-replayer-default" --desired-count 0 > /dev/null 2>&1 +} + # One-time required service-linked-role creation for AWS accounts which do not have these roles create_service_linked_roles () { aws iam create-service-linked-role --aws-service-name opensearchservice.amazonaws.com @@ -60,9 +84,13 @@ usage() { echo "an OpenSearch Service Domain as the target cluster, and the Migration infrastructure for simulating a migration from source to target." echo "" echo "Usage: " - echo " ./awsE2ESolutionSetup.sh [--create-service-linked-roles] [--bootstrap-region] [--enable-capture-proxy]" + echo " ./awsE2ESolutionSetup.sh [--migrations-git-url] [--migrations-git-branch] [--stage] [--create-service-linked-roles] [--bootstrap-region] [--enable-capture-proxy] [--run-post-actions]" echo "" echo "Options:" + echo " --migrations-git-url The Github http url used for building the capture proxy on setups with a dedicated source cluster, default is 'https://github.com/opensearch-project/opensearch-migrations.git'." + echo " --migrations-git-branch The Github branch associated with the 'git-url' to pull from, default is 'main'." + echo " --stage The stage name to use for naming/grouping of AWS deployment resources, default is 'aws-integ'." + echo " --run-post-actions Flag to enable only running post test actions for cleaning up and recording a test run." echo " --create-service-linked-roles If included, will attempt to create required service linked roles for the AWS account" echo " --bootstrap-region If included, will attempt to CDK bootstrap the region to allow CDK deployments" echo " --enable-capture-proxy If included, will attempt to enable the capture proxy on all source nodes" @@ -70,10 +98,14 @@ usage() { exit 1 } -stage='aws-integ' +STAGE='aws-integ' +RUN_POST_ACTIONS=false CREATE_SLR=false BOOTSTRAP_REGION=false ENABLE_CAPTURE_PROXY=false +MIGRATIONS_GIT_URL='https://github.com/opensearch-project/opensearch-migrations.git' +MIGRATIONS_GIT_BRANCH='main' + while [[ $# -gt 0 ]]; do case $1 in --create-service-linked-roles) @@ -84,10 +116,29 @@ while [[ $# -gt 0 ]]; do BOOTSTRAP_REGION=true shift # past argument ;; + --run-post-actions) + RUN_POST_ACTIONS=true + shift # past argument + ;; --enable-capture-proxy) ENABLE_CAPTURE_PROXY=true shift # past argument ;; + --migrations-git-url) + MIGRATIONS_GIT_URL="$2" + shift # past argument + shift # past value + ;; + --migrations-git-branch) + MIGRATIONS_GIT_BRANCH="$2" + shift # past argument + shift # past value + ;; + --stage) + STAGE="$2" + shift # past argument + shift # past value + ;; -h|--help) usage ;; @@ -101,15 +152,19 @@ while [[ $# -gt 0 ]]; do esac done +if [ "$RUN_POST_ACTIONS" = true ] ; then + restore_and_record "$STAGE" + exit 0 +fi + if [ "$CREATE_SLR" = true ] ; then create_service_linked_roles fi -# TODO allow spaces in CDK context block when passing as CLI param # Store CDK context for migration solution deployment in variable read -r -d '' cdk_context << EOM { - "stage": "$stage", + "stage": "$STAGE", "vpcId": "", "engineVersion": "OS_2.9", "domainName": "opensearch-cluster-aws-integ", @@ -118,6 +173,7 @@ read -r -d '' cdk_context << EOM "mskBrokerNodeCount": 2, "openAccessPolicyEnabled": true, "domainRemovalPolicy": "DESTROY", + "trafficReplayerExtraArgs": "--speedup-factor 10.0", "migrationAnalyticsServiceEnabled": false, "fetchMigrationEnabled": true, "sourceClusterEndpoint": "", @@ -133,25 +189,24 @@ if [ "$BOOTSTRAP_REGION" = true ] ; then bootstrap_region fi # Deploy source cluster on EC2 instances -cdk deploy "*" --require-approval never --c suffix="Migration-Source" --c distVersion="7.10.2" --c distributionUrl="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.10.2-linux-x86_64.tar.gz" --c captureProxyEnabled=true --c captureProxyTarUrl="https://github.com/opensearch-project/opensearch-migrations/releases/download/1.0.0/trafficCaptureProxyServer.x86_64.tar" --c securityDisabled=true --c minDistribution=false --c cpuArch="x64" --c isInternal=true --c singleNodeCluster=true --c networkAvailabilityZones=2 --c dataNodeCount=2 --c managerNodeCount=0 --c serverAccessType="ipv4" --c restrictServerAccessTo="0.0.0.0/0" +cdk deploy "*" --require-approval never --c suffix="Migration-Source" --c distVersion="7.10.2" --c distributionUrl="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.10.2-linux-x86_64.tar.gz" --c captureProxyEnabled=true --c securityDisabled=true --c minDistribution=false --c cpuArch="x64" --c isInternal=true --c singleNodeCluster=true --c networkAvailabilityZones=2 --c dataNodeCount=2 --c managerNodeCount=0 --c serverAccessType="ipv4" --c restrictServerAccessTo="0.0.0.0/0" source_endpoint=$(aws cloudformation describe-stacks --stack-name opensearch-infra-stack-Migration-Source --query "Stacks[0].Outputs[?OutputKey==\`loadbalancerurl\`].OutputValue" --output text) echo $source_endpoint vpc_id=$(aws cloudformation describe-stacks --stack-name opensearch-network-stack --query "Stacks[0].Outputs[?contains(OutputValue, 'vpc')].OutputValue" --output text) echo $vpc_id cdk_context=$(echo "${cdk_context//$vpc_id}") -cdk_context=$(echo "${cdk_context//http://${source_endpoint}}") +cdk_context=$(echo "${cdk_context//http://${source_endpoint}:19200}") cdk_context=$(echo $cdk_context | jq '@json') +# TODO Further verify space escaping for JSON +# Escape spaces for CDK JSON parsing to handle +cdk_context=$(echo "${cdk_context/ /\\u0020}") +echo $cdk_context cd ../../deployment/cdk/opensearch-service-migration ./buildDockerImages.sh npm install cdk deploy "*" --c aws-existing-source=$cdk_context --c contextId=aws-existing-source --require-approval never --concurrency 3 if [ "$ENABLE_CAPTURE_PROXY" = true ] ; then - prepare_source_nodes_for_capture $stage -fi - -# Kickoff integration tests -#task_arn=$(aws ecs list-tasks --cluster migration-${stage}-ecs-cluster --family "migration-${stage}-migration-console" | jq --raw-output '.taskArns[0]') -#echo "aws ecs execute-command --cluster 'migration-${stage}-ecs-cluster' --task '${task_arn}' --container 'migration-console' --interactive --command '/bin/bash'" -#aws ecs execute-command --cluster "migration-${stage}-ecs-cluster" --task "${task_arn}" --container "migration-console" --interactive --command "./setupIntegTests.sh https://github.com/opensearch-project/opensearch-migrations.git main" \ No newline at end of file + prepare_source_nodes_for_capture "$STAGE" +fi \ No newline at end of file diff --git a/test/awsRunIntegTests.sh b/test/awsRunIntegTests.sh new file mode 100755 index 000000000..09f305866 --- /dev/null +++ b/test/awsRunIntegTests.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +usage() { + echo "" + echo "Script to run integrations tests on AWS Migration Console" + echo "" + echo "Usage: " + echo " ./awsRunIntegTests.sh [--unique-id] [--migrations-git-url] [--migrations-git-branch] [--stage]" + echo "" + echo "Options:" + echo " --unique-id Identifier for labeling integ test artifacts, e.g. 'full_run_123'." + echo " --migrations-git-url The Github http url used for pulling the integration tests onto the migration console, default is 'https://github.com/opensearch-project/opensearch-migrations.git'." + echo " --migrations-git-branch The Github branch associated with the 'git-url' to pull from, default is 'main'." + echo " --stage The stage used for CDK deployment, default is 'aws-integ'." + echo "" + exit 1 +} + +epoch_seconds=$(date +%s) +UNIQUE_ID="test_${epoch_seconds}_1" +STAGE='aws-integ' +MIGRATIONS_GIT_URL='https://github.com/opensearch-project/opensearch-migrations.git' +MIGRATIONS_GIT_BRANCH='main' + +while [[ $# -gt 0 ]]; do + case $1 in + --unique-id) + UNIQUE_ID="$2" + shift # past argument + shift # past value + ;; + --stage) + STAGE="$2" + shift # past argument + shift # past value + ;; + --migrations-git-url) + MIGRATIONS_GIT_URL="$2" + shift # past argument + shift # past value + ;; + --migrations-git-branch) + MIGRATIONS_GIT_BRANCH="$2" + shift # past argument + shift # past value + ;; + -h|--help) + usage + ;; + -*) + echo "Unknown option $1" + usage + ;; + *) + shift # past argument + ;; + esac +done + +task_arn=$(aws ecs list-tasks --cluster migration-${STAGE}-ecs-cluster --family "migration-${STAGE}-migration-console" | jq --raw-output '.taskArns[0]') +# Delete and re-create topic +kafka_brokers=$(aws ssm get-parameter --name "/migration/${STAGE}/default/kafkaBrokers" --query 'Parameter.Value' --output text) +unbuffer aws ecs execute-command --cluster "migration-${STAGE}-ecs-cluster" --task "${task_arn}" --container "migration-console" --interactive --command "./kafka-tools/kafka/bin/kafka-topics.sh --bootstrap-server ${kafka_brokers} --delete --topic logging-traffic-topic --command-config kafka-tools/aws/msk-iam-auth.properties" +echo "Done deleting 'logging-traffic-topic'" +unbuffer aws ecs execute-command --cluster "migration-${STAGE}-ecs-cluster" --task "${task_arn}" --container "migration-console" --interactive --command "./kafka-tools/kafka/bin/kafka-topics.sh --bootstrap-server ${kafka_brokers} --create --topic logging-traffic-topic --command-config kafka-tools/aws/msk-iam-auth.properties" +echo "Done creating 'logging-traffic-topic'" + +# Remove all non-system indices +source_lb_endpoint=$(aws cloudformation describe-stacks --stack-name opensearch-infra-stack-Migration-Source --query "Stacks[0].Outputs[?OutputKey==\`loadbalancerurl\`].OutputValue" --output text) +source_endpoint="http://${source_lb_endpoint}:19200" +proxy_endpoint="http://${source_lb_endpoint}:9200" +target_endpoint=$(aws ssm get-parameter --name "/migration/${STAGE}/default/osClusterEndpoint" --query 'Parameter.Value' --output text) +echo "Clearing non-system source indices" +unbuffer aws ecs execute-command --cluster "migration-${STAGE}-ecs-cluster" --task "${task_arn}" --container "migration-console" --interactive --command "curl -XDELETE ${source_endpoint}/*,-.*,-searchguard*,-sg7*" +echo "Clearing non-system target indices" +unbuffer aws ecs execute-command --cluster "migration-${STAGE}-ecs-cluster" --task "${task_arn}" --container "migration-console" --interactive --command "curl -XDELETE ${target_endpoint}/*,-.*" + +echo "Print initial source and target indices after clearing indices: " +unbuffer aws ecs execute-command --cluster "migration-${STAGE}-ecs-cluster" --task "${task_arn}" --container "migration-console" --interactive --command "./catIndices.sh --source_endpoint ${source_endpoint} --source_no_auth --target_no_auth" + +# Spin up Replayer container and wait for service to be stable +aws ecs update-service --cluster "migration-${STAGE}-ecs-cluster" --service "migration-${STAGE}-traffic-replayer-default" --desired-count 1 > /dev/null 2>&1 +echo "Waiting for Replayer to be stable..." +aws ecs wait services-stable --cluster "migration-${STAGE}-ecs-cluster" --service "migration-${STAGE}-traffic-replayer-default" + +# Kickoff integration tests +set -o xtrace +unbuffer aws ecs execute-command --cluster "migration-${STAGE}-ecs-cluster" --task "${task_arn}" --container "migration-console" --interactive --command "./setupIntegTests.sh ${MIGRATIONS_GIT_URL} ${MIGRATIONS_GIT_BRANCH} ${source_endpoint} ${proxy_endpoint} ${UNIQUE_ID}" +set +o xtrace +test_output=$(unbuffer aws ecs execute-command --cluster "migration-${STAGE}-ecs-cluster" --task "${task_arn}" --container "migration-console" --interactive --command "awk '/failures/ && /errors/' /root/integ-tests/test/reports/${UNIQUE_ID}.xml") +echo "Fetch integ test summary: " +echo "$test_output" +failure_output=$(echo "$test_output" | grep -o "failures=\"0\"") +if [ -z "$failure_output" ]; then + echo "Failed test detected in output, failing step" + exit 1 +fi +errors_output=$(echo "$test_output" | grep -o "errors=\"0\"") +if [ -z "$errors_output" ]; then + echo "Errored test detected in output, failing step" + exit 1 +fi +exit 0 \ No newline at end of file