diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index daf820c6fe..1e772923ee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,13 +6,16 @@ on: release-major-tag: description: 'Whether to create major tag of docker image or not. This will create a tag such as 2.3 which points to this version.' required: true + type: boolean release-latest-tag: description: > 'Whether to create latest tag of docker image or not. This will update the latest tag to point to this version. You should set this when releasing the latest version, but not patches to old versions.' required: true + type: boolean permissions: id-token: write + contents: write jobs: build: @@ -104,7 +107,7 @@ jobs: - name: Smoke Test Tarball Files run: ./release/smoke-tests/run-tarball-files-smoke-tests.sh -v ${{ env.version }} -u ${{ secrets.ARCHIVES_PUBLIC_URL }} -n ${{ github.run_number }} -i ${{ matrix.image }} -t ${{ matrix.archive }} - protomote: + promote: runs-on: ubuntu-latest if: success() || failure() needs: [build, validate-docker, validate-archive] diff --git a/jenkins/release.jenkinsFile b/jenkins/release.jenkinsFile new file mode 100644 index 0000000000..839b4bba15 --- /dev/null +++ b/jenkins/release.jenkinsFile @@ -0,0 +1,299 @@ +lib = library(identifier: 'jenkins@5.6.1', retriever: modernSCM([ + $class: 'GitSCMSource', + remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', +])) + +def VERSION = '' +def DATA_PREPPER_BUILD_NUMBER = '' +def RELEASE_MAJOR_TAG = '' +def RELEASE_LATEST_TAG = '' + +pipeline { + options { + timeout(time: 1, unit: 'HOURS') + } + agent none + triggers { + GenericTrigger( + genericVariables: [ + [key: 'ref', value: ('$.release.tag_name')], + [key: 'tag', value: '$.release.name'], + [key: 'action', value: '$.action'], + [key: 'isDraft', value: '$.release.draft'], + [key: 'release_url', value: '$.release.url'], + [key: 'assets_url', value: '$.release.assets_url'] + ], + tokenCredentialId: 'jenkins-data-prepper-generic-webhook-token', + causeString: 'Triggered by draft release on data-prepper repository', + printContributedVariables: false, + printPostContent: false, + regexpFilterText: ('$isDraft $action'), + regexpFilterExpression: ('^true created$') + ) + } + environment { + DATA_PREPPER_ARTIFACT_STAGING_SITE = credentials('jenkins-data-prepper-artifact-staging-site') + DATA_PREPPER_STAGING_CONTAINER_REPOSITORY = credentials('jenkins-data-prepper-staging-container-repository') + } + stages { + stage('Get release paramters') { + agent { + docker { + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' + image 'opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3' + args '-e JAVA_HOME=/opt/java/openjdk-11' + registryUrl 'https://public.ecr.aws/' + alwaysPull true + } + } + steps { + script { + if ("$assets_url" != '') { + withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN')]) { + String assets = sh( + script: "curl -H 'Accept: application/vnd.github+json' -H 'Authorization: Bearer ${GITHUB_TOKEN}' ${assets_url}", + returnStdout: true + ) + String assetUrl = null + def parsedJson = readJSON text: assets + def assetName = 'release-description.yaml' + parsedJson.each { item -> + if (item.name == assetName) { + assetUrl = item.url + } + } + echo "Downloading release-description.yaml from $assetUrl" + sh "curl -J -L -H 'Accept: application/octet-stream' -H 'Authorization: Bearer ${GITHUB_TOKEN}' ${assetUrl} -o ${WORKSPACE}/release-description.yaml" + } + def yamlFile = readYaml(file: "${WORKSPACE}/release-description.yaml") + VERSION = yamlFile.version + DATA_PREPPER_BUILD_NUMBER = yamlFile.build_number + RELEASE_MAJOR_TAG = yamlFile.release_major_tag + RELEASE_LATEST_TAG = yamlFile.release_latest_tag + + if (isNullOrEmpty(VERSION) || isNullOrEmpty(DATA_PREPPER_BUILD_NUMBER) || isNullOrEmpty(RELEASE_MAJOR_TAG) || isNullOrEmpty(RELEASE_LATEST_TAG)) { + currentBuild.result = 'ABORTED' + error('Value of version, build_number, release_major_tag, release_latest_tag cannot be null or empty. Please check release-description.yaml') + } + } else { + currentBuild.result = 'ABORTED' + error("$assets_url is empty. Unable to download release-description.yaml") + } + } + } + } + stage('Promote Archives') { + agent { + docker { + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' + image 'opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3' + args '-e JAVA_HOME=/opt/java/openjdk-11' + registryUrl 'https://public.ecr.aws/' + alwaysPull true + } + } + stages { + stage('Download Archives') { + steps { + script { + archivePath = "${DATA_PREPPER_ARTIFACT_STAGING_SITE}/${VERSION}/${DATA_PREPPER_BUILD_NUMBER}/archive" + + dir('archive') { + sh "curl -sSL ${archivePath}/opensearch-data-prepper-${VERSION}-linux-x64.tar.gz -o opensearch-data-prepper-${VERSION}-linux-x64.tar.gz" + sh "curl -sSL ${archivePath}/opensearch-data-prepper-jdk-${VERSION}-linux-x64.tar.gz -o opensearch-data-prepper-jdk-${VERSION}-linux-x64.tar.gz" + } + } + } + } + stage('Sign and Release Archives') { + steps { + script { + publishToArtifactsProdBucket( + assumedRoleName: 'data-prepper-artifacts-upload-role', + source: "${env.WORKSPACE}/archive", + destination: "data-prepper/${VERSION}/", + signingPlatform: 'linux', + sigType: '.sig', + sigOverwrite: true + ) + } + } + } + } + post() { + always { + script { + postCleanup() + } + } + } + } + stage('Promote Docker') { + agent { + docker { + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' + image 'docker/library/alpine:3' + registryUrl 'https://public.ecr.aws/' + alwaysPull true + } + } + stages { + stage('Copy Docker Image to DockerHub') { + steps { + script { + def dockerCopyHub = + build job: 'docker-copy', + parameters: [ + string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"), + string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"), + string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'opensearchproject'), + string(name: 'DESTINATION_IMAGE', value: "data-prepper:${VERSION}") + ] + + if (RELEASE_MAJOR_TAG) { + def majorVersion = VERSION.tokenize('.')[0].trim() + def dockerCopyHubMajor = + build job: 'docker-copy', + parameters: [ + string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"), + string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"), + string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'opensearchproject'), + string(name: 'DESTINATION_IMAGE', value: "data-prepper:${majorVersion}") + ] + } + + if (RELEASE_LATEST_TAG) { + def dockerCopyHubLatest = + build job: 'docker-copy', + parameters: [ + string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"), + string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"), + string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'opensearchproject'), + string(name: 'DESTINATION_IMAGE', value: 'data-prepper:latest') + ] + } + } + } + } + stage('Copy Docker Image to ECR') { + steps { + script { + def dockerCopyECR = + build job: 'docker-copy', + parameters: [ + string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"), + string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"), + string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'public.ecr.aws/opensearchproject'), + string(name: 'DESTINATION_IMAGE', value: "data-prepper:${VERSION}") + ] + + if (RELEASE_MAJOR_TAG) { + def majorVersion = VERSION.tokenize('.')[0].trim() + def dockerCopyECRMajor = + build job: 'docker-copy', + parameters: [ + string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"), + string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"), + string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'public.ecr.aws/opensearchproject'), + string(name: 'DESTINATION_IMAGE', value: "data-prepper:${majorVersion}") + ] + } + + if (RELEASE_LATEST_TAG) { + def dockerCopyECRLatest = + build job: 'docker-copy', + parameters: [ + string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"), + string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"), + string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'public.ecr.aws/opensearchproject'), + string(name: 'DESTINATION_IMAGE', value: 'data-prepper:latest') + ] + } + } + } + } + } + post() { + always { + script { + postCleanup() + } + } + } + } + stage('Promote Maven') { + agent { + docker { + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' + image 'opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3' + args '-e JAVA_HOME=/opt/java/openjdk-11' + registryUrl 'https://public.ecr.aws/' + alwaysPull true + } + } + stages { + stage('Download Maven Artifacts') { + steps { + script { + mavenPath = "${DATA_PREPPER_ARTIFACT_STAGING_SITE}/${VERSION}/${DATA_PREPPER_BUILD_NUMBER}/maven" + group = 'org/opensearch/dataprepper' + artifacts = ['data-prepper-api'] + fileTypes = ['-javadoc.jar', '.jar', '.pom', '-sources.jar', '.module'] + checksums = ['', '.md5', '.sha1', '.sha256', '.sha512'] + + downloadArtifacts() + } + } + } + stage('Publish To Maven') { + steps { + script { + publishToMaven( + signingArtifactsPath: "${WORKSPACE}/maven", + mavenArtifactsPath: "${WORKSPACE}/maven", + autoPublish: true + ) + } + } + } + } + post() { + always { + script { + postCleanup() + } + } + } + } + } + post { + success { + node('Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host') { + script { + if (release_url != null) { + withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN')]) { + sh "curl -X PATCH -H 'Accept: application/vnd.github+json' -H 'Authorization: Bearer ${GITHUB_TOKEN}' ${release_url} -d '{\"tag_name\":\"${TAG}\",\"draft\":false,\"prerelease\":false}'" + } + } + } + } + } + } +} + +def downloadArtifacts() { + dir('maven') { + for (artifact in artifacts) { + sh "mkdir -p ${group}/${artifact}/${VERSION}" + for (fileType in fileTypes) { + for (checksum in checksums) { + sh "curl -sSL ${mavenPath}/${group}/${artifact}/${VERSION}/${artifact}-${VERSION}${fileType}${checksum} -o ${group}/${artifact}/${VERSION}/${artifact}-${VERSION}${fileType}${checksum}" + } + } + } + } +} + +def isNullOrEmpty(str) { + return (str == null || str == '') +}