Add jenkins release workflow and refactor GHA workflow (#3204)

Signed-off-by: Sayali Gaikawad <[email protected]>

.github/workflows/release.yml

@@ -6,13 +6,16 @@ on:
       release-major-tag:
         description: 'Whether to create major tag of docker image or not. This will create a tag such as 2.3 which points to this version.'
         required: true
+        type: boolean
       release-latest-tag:
         description: >
           'Whether to create latest tag of docker image or not. This will update the latest tag to point to this version. You should set this when releasing the latest version, but not patches to old versions.'
         required: true
+        type: boolean
 
 permissions:
   id-token: write
+  contents: write
 
 jobs:
   build:
@@ -104,7 +107,7 @@ jobs:
       - name: Smoke Test Tarball Files
         run: ./release/smoke-tests/run-tarball-files-smoke-tests.sh -v ${{ env.version }} -u ${{ secrets.ARCHIVES_PUBLIC_URL }} -n ${{ github.run_number }} -i ${{ matrix.image }} -t ${{ matrix.archive }}
 
-  protomote:
+  promote:
     runs-on: ubuntu-latest
     if: success() || failure()
     needs: [build, validate-docker, validate-archive]

jenkins/release.jenkinsFile

@@ -0,0 +1,299 @@
+lib = library(identifier: '[email protected]', retriever: modernSCM([
+    $class: 'GitSCMSource',
+    remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
+]))
+
+def VERSION = ''
+def DATA_PREPPER_BUILD_NUMBER = ''
+def RELEASE_MAJOR_TAG = ''
+def RELEASE_LATEST_TAG = ''
+
+pipeline {
+    options {
+        timeout(time: 1, unit: 'HOURS')
+    }
+    agent none
+    triggers {
+        GenericTrigger(
+            genericVariables: [
+                [key: 'ref', value: ('$.release.tag_name')],
+                [key: 'tag', value: '$.release.name'],
+                [key: 'action', value: '$.action'],
+                [key: 'isDraft', value: '$.release.draft'],
+                [key: 'release_url', value: '$.release.url'],
+                [key: 'assets_url', value: '$.release.assets_url']
+            ],
+            tokenCredentialId: 'jenkins-data-prepper-generic-webhook-token',
+            causeString: 'Triggered by draft release on data-prepper repository',
+            printContributedVariables: false,
+            printPostContent: false,
+            regexpFilterText: ('$isDraft $action'),
+            regexpFilterExpression: ('^true created$')
+        )
+    }
+    environment {
+        DATA_PREPPER_ARTIFACT_STAGING_SITE = credentials('jenkins-data-prepper-artifact-staging-site')
+        DATA_PREPPER_STAGING_CONTAINER_REPOSITORY = credentials('jenkins-data-prepper-staging-container-repository')
+    }
+    stages {
+        stage('Get release paramters') {
+            agent {
+                docker {
+                    label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host'
+                    image 'opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3'
+                    args '-e JAVA_HOME=/opt/java/openjdk-11'
+                    registryUrl 'https://public.ecr.aws/'
+                    alwaysPull true
+                }
+            }
+            steps {
+                script {
+                    if ("$assets_url" != '') {
+                        withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN')]) {
+                            String assets = sh(
+                                script: "curl -H 'Accept: application/vnd.github+json' -H 'Authorization: Bearer ${GITHUB_TOKEN}' ${assets_url}",
+                                returnStdout: true
+                            )
+                            String assetUrl = null
+                            def parsedJson = readJSON text: assets
+                            def assetName = 'release-description.yaml'
+                            parsedJson.each { item ->
+                                if (item.name == assetName) {
+                                    assetUrl = item.url
+                                }
+                            }
+                            echo "Downloading release-description.yaml from $assetUrl"
+                            sh "curl -J -L -H 'Accept: application/octet-stream' -H 'Authorization: Bearer ${GITHUB_TOKEN}' ${assetUrl} -o ${WORKSPACE}/release-description.yaml"
+                        }
+                        def yamlFile = readYaml(file: "${WORKSPACE}/release-description.yaml")
+                        VERSION = yamlFile.version
+                        DATA_PREPPER_BUILD_NUMBER = yamlFile.build_number
+                        RELEASE_MAJOR_TAG = yamlFile.release_major_tag
+                        RELEASE_LATEST_TAG = yamlFile.release_latest_tag
+
+                        if (isNullOrEmpty(VERSION) || isNullOrEmpty(DATA_PREPPER_BUILD_NUMBER) || isNullOrEmpty(RELEASE_MAJOR_TAG) || isNullOrEmpty(RELEASE_LATEST_TAG)) {
+                            currentBuild.result = 'ABORTED'
+                            error('Value of version, build_number, release_major_tag, release_latest_tag cannot be null or empty. Please check release-description.yaml')
+                        }
+                     } else {
+                        currentBuild.result = 'ABORTED'
+                        error("$assets_url is empty. Unable to download release-description.yaml")
+                    }
+                }
+            }
+        }
+        stage('Promote Archives') {
+            agent {
+                docker {
+                    label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host'
+                    image 'opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3'
+                    args '-e JAVA_HOME=/opt/java/openjdk-11'
+                    registryUrl 'https://public.ecr.aws/'
+                    alwaysPull true
+                }
+            }
+            stages {
+                stage('Download Archives') {
+                    steps {
+                        script {
+                            archivePath = "${DATA_PREPPER_ARTIFACT_STAGING_SITE}/${VERSION}/${DATA_PREPPER_BUILD_NUMBER}/archive"
+
+                            dir('archive') {
+                                sh "curl -sSL ${archivePath}/opensearch-data-prepper-${VERSION}-linux-x64.tar.gz -o opensearch-data-prepper-${VERSION}-linux-x64.tar.gz"
+                                sh "curl -sSL ${archivePath}/opensearch-data-prepper-jdk-${VERSION}-linux-x64.tar.gz -o opensearch-data-prepper-jdk-${VERSION}-linux-x64.tar.gz"
+                            }
+                        }
+                    }
+                }
+                stage('Sign and Release Archives') {
+                    steps {
+                        script {
+                            publishToArtifactsProdBucket(
+                                assumedRoleName: 'data-prepper-artifacts-upload-role',
+                                source: "${env.WORKSPACE}/archive",
+                                destination: "data-prepper/${VERSION}/",
+                                signingPlatform: 'linux',
+                                sigType: '.sig',
+                                sigOverwrite: true
+                            )
+                        }
+                    }
+                }
+            }
+            post() {
+                always {
+                    script {
+                        postCleanup()
+                    }
+                }
+            }
+        }
+        stage('Promote Docker') {
+            agent {
+                docker {
+                    label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host'
+                    image 'docker/library/alpine:3'
+                    registryUrl 'https://public.ecr.aws/'
+                    alwaysPull true
+                }
+            }
+            stages {
+                stage('Copy Docker Image to DockerHub') {
+                    steps {
+                        script {
+                            def dockerCopyHub =
+                                build job: 'docker-copy',
+                                parameters: [
+                                    string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
+                                    string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
+                                    string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'opensearchproject'),
+                                    string(name: 'DESTINATION_IMAGE', value: "data-prepper:${VERSION}")
+                                ]
+
+                            if (RELEASE_MAJOR_TAG) {
+                                def majorVersion = VERSION.tokenize('.')[0].trim()
+                                def dockerCopyHubMajor =
+                                    build job: 'docker-copy',
+                                    parameters: [
+                                        string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
+                                        string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
+                                        string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'opensearchproject'),
+                                        string(name: 'DESTINATION_IMAGE', value: "data-prepper:${majorVersion}")
+                                    ]
+                            }
+
+                            if (RELEASE_LATEST_TAG) {
+                                def dockerCopyHubLatest =
+                                    build job: 'docker-copy',
+                                    parameters: [
+                                        string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
+                                        string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
+                                        string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'opensearchproject'),
+                                        string(name: 'DESTINATION_IMAGE', value: 'data-prepper:latest')
+                                    ]
+                            }
+                        }
+                    }
+                }
+                stage('Copy Docker Image to ECR') {
+                    steps {
+                        script {
+                            def dockerCopyECR =
+                                build job: 'docker-copy',
+                                parameters: [
+                                    string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
+                                    string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
+                                    string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'public.ecr.aws/opensearchproject'),
+                                    string(name: 'DESTINATION_IMAGE', value: "data-prepper:${VERSION}")
+                                ]
+
+                            if (RELEASE_MAJOR_TAG) {
+                                def majorVersion = VERSION.tokenize('.')[0].trim()
+                                def dockerCopyECRMajor =
+                                    build job: 'docker-copy',
+                                    parameters: [
+                                        string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
+                                        string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
+                                        string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'public.ecr.aws/opensearchproject'),
+                                        string(name: 'DESTINATION_IMAGE', value: "data-prepper:${majorVersion}")
+                                    ]
+                            }
+
+                            if (RELEASE_LATEST_TAG) {
+                                def dockerCopyECRLatest =
+                                    build job: 'docker-copy',
+                                    parameters: [
+                                        string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
+                                        string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
+                                        string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'public.ecr.aws/opensearchproject'),
+                                        string(name: 'DESTINATION_IMAGE', value: 'data-prepper:latest')
+                                    ]
+                            }
+                        }
+                    }
+                }
+            }
+            post() {
+                always {
+                    script {
+                        postCleanup()
+                    }
+                }
+            }
+        }
+        stage('Promote Maven') {
+            agent {
+                docker {
+                    label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host'
+                    image 'opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3'
+                    args '-e JAVA_HOME=/opt/java/openjdk-11'
+                    registryUrl 'https://public.ecr.aws/'
+                    alwaysPull true
+                }
+            }
+            stages {
+                stage('Download Maven Artifacts') {
+                    steps {
+                        script {
+                            mavenPath = "${DATA_PREPPER_ARTIFACT_STAGING_SITE}/${VERSION}/${DATA_PREPPER_BUILD_NUMBER}/maven"
+                            group = 'org/opensearch/dataprepper'
+                            artifacts = ['data-prepper-api']
+                            fileTypes = ['-javadoc.jar', '.jar', '.pom', '-sources.jar', '.module']
+                            checksums = ['', '.md5', '.sha1', '.sha256', '.sha512']
+
+                            downloadArtifacts()
+                        }
+                    }
+                }
+                stage('Publish To Maven') {
+                    steps {
+                        script {
+                            publishToMaven(
+                                signingArtifactsPath: "${WORKSPACE}/maven",
+                                mavenArtifactsPath: "${WORKSPACE}/maven",
+                                autoPublish: true
+                            )
+                        }
+                    }
+                }
+            }
+            post() {
+                always {
+                    script {
+                        postCleanup()
+                    }
+                }
+            }
+        }
+    }
+    post {
+        success {
+            node('Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host') {
+                script {
+                    if (release_url != null) {
+                        withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN')]) {
+                            sh "curl -X PATCH -H 'Accept: application/vnd.github+json' -H 'Authorization: Bearer ${GITHUB_TOKEN}' ${release_url} -d '{\"tag_name\":\"${TAG}\",\"draft\":false,\"prerelease\":false}'"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+def downloadArtifacts() {
+    dir('maven') {
+        for (artifact in artifacts) {
+            sh "mkdir -p ${group}/${artifact}/${VERSION}"
+            for (fileType in fileTypes) {
+                for (checksum in checksums) {
+                    sh "curl -sSL ${mavenPath}/${group}/${artifact}/${VERSION}/${artifact}-${VERSION}${fileType}${checksum} -o ${group}/${artifact}/${VERSION}/${artifact}-${VERSION}${fileType}${checksum}"
+                }
+            }
+        }
+    }
+}
+
+def isNullOrEmpty(str) {
+    return (str == null || str == '')
+}