Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add jenkins release workflow and refactor GHA workflow #3204

Merged
merged 1 commit into from
Aug 22, 2023
Merged

Add jenkins release workflow and refactor GHA workflow #3204

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .github/workflows/release.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,16 @@ on:
release-major-tag:
description: 'Whether to create major tag of docker image or not. This will create a tag such as 2.3 which points to this version.'
required: true
type: boolean
release-latest-tag:
description: >
'Whether to create latest tag of docker image or not. This will update the latest tag to point to this version. You should set this when releasing the latest version, but not patches to old versions.'
required: true
type: boolean

permissions:
id-token: write
contents: write

jobs:
build:
Expand Down Expand Up @@ -104,7 +107,7 @@ jobs:
- name: Smoke Test Tarball Files
run: ./release/smoke-tests/run-tarball-files-smoke-tests.sh -v ${{ env.version }} -u ${{ secrets.ARCHIVES_PUBLIC_URL }} -n ${{ github.run_number }} -i ${{ matrix.image }} -t ${{ matrix.archive }}

protomote:
promote:
runs-on: ubuntu-latest
if: success() || failure()
needs: [build, validate-docker, validate-archive]
Expand Down
299 changes: 299 additions & 0 deletions jenkins/release.jenkinsFile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,299 @@
lib = library(identifier: '[email protected]', retriever: modernSCM([
$class: 'GitSCMSource',
remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
]))

def VERSION = ''
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am curious as to the use of a global variable here. Is there not a way to pass the parameter from the trigger to the job directly?

def DATA_PREPPER_BUILD_NUMBER = ''
def RELEASE_MAJOR_TAG = ''
def RELEASE_LATEST_TAG = ''

pipeline {
options {
timeout(time: 1, unit: 'HOURS')
}
agent none
triggers {
GenericTrigger(
genericVariables: [
[key: 'ref', value: ('$.release.tag_name')],
[key: 'tag', value: '$.release.name'],
[key: 'action', value: '$.action'],
[key: 'isDraft', value: '$.release.draft'],
[key: 'release_url', value: '$.release.url'],
[key: 'assets_url', value: '$.release.assets_url']
],
tokenCredentialId: 'jenkins-data-prepper-generic-webhook-token',
causeString: 'Triggered by draft release on data-prepper repository',
printContributedVariables: false,
printPostContent: false,
regexpFilterText: ('$isDraft $action'),
regexpFilterExpression: ('^true created$')
)
}
environment {
DATA_PREPPER_ARTIFACT_STAGING_SITE = credentials('jenkins-data-prepper-artifact-staging-site')
DATA_PREPPER_STAGING_CONTAINER_REPOSITORY = credentials('jenkins-data-prepper-staging-container-repository')
}
stages {
stage('Get release paramters') {
agent {
docker {
label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host'
image 'opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3'
args '-e JAVA_HOME=/opt/java/openjdk-11'
registryUrl 'https://public.ecr.aws/'
alwaysPull true
}
}
steps {
script {
if ("$assets_url" != '') {
withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN')]) {
String assets = sh(
script: "curl -H 'Accept: application/vnd.github+json' -H 'Authorization: Bearer ${GITHUB_TOKEN}' ${assets_url}",
returnStdout: true
)
String assetUrl = null
def parsedJson = readJSON text: assets
def assetName = 'release-description.yaml'
parsedJson.each { item ->
if (item.name == assetName) {
assetUrl = item.url
}
}
echo "Downloading release-description.yaml from $assetUrl"
sh "curl -J -L -H 'Accept: application/octet-stream' -H 'Authorization: Bearer ${GITHUB_TOKEN}' ${assetUrl} -o ${WORKSPACE}/release-description.yaml"
}
def yamlFile = readYaml(file: "${WORKSPACE}/release-description.yaml")
VERSION = yamlFile.version
DATA_PREPPER_BUILD_NUMBER = yamlFile.build_number
RELEASE_MAJOR_TAG = yamlFile.release_major_tag
RELEASE_LATEST_TAG = yamlFile.release_latest_tag

if (isNullOrEmpty(VERSION) || isNullOrEmpty(DATA_PREPPER_BUILD_NUMBER) || isNullOrEmpty(RELEASE_MAJOR_TAG) || isNullOrEmpty(RELEASE_LATEST_TAG)) {
currentBuild.result = 'ABORTED'
error('Value of version, build_number, release_major_tag, release_latest_tag cannot be null or empty. Please check release-description.yaml')
}
} else {
currentBuild.result = 'ABORTED'
error("$assets_url is empty. Unable to download release-description.yaml")
}
}
}
}
stage('Promote Archives') {
agent {
docker {
label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host'
image 'opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3'
args '-e JAVA_HOME=/opt/java/openjdk-11'
registryUrl 'https://public.ecr.aws/'
alwaysPull true
}
}
stages {
stage('Download Archives') {
steps {
script {
archivePath = "${DATA_PREPPER_ARTIFACT_STAGING_SITE}/${VERSION}/${DATA_PREPPER_BUILD_NUMBER}/archive"

dir('archive') {
sh "curl -sSL ${archivePath}/opensearch-data-prepper-${VERSION}-linux-x64.tar.gz -o opensearch-data-prepper-${VERSION}-linux-x64.tar.gz"
sh "curl -sSL ${archivePath}/opensearch-data-prepper-jdk-${VERSION}-linux-x64.tar.gz -o opensearch-data-prepper-jdk-${VERSION}-linux-x64.tar.gz"
}
}
}
}
stage('Sign and Release Archives') {
steps {
script {
publishToArtifactsProdBucket(
assumedRoleName: 'data-prepper-artifacts-upload-role',
source: "${env.WORKSPACE}/archive",
destination: "data-prepper/${VERSION}/",
signingPlatform: 'linux',
sigType: '.sig',
sigOverwrite: true
)
}
}
}
}
post() {
always {
script {
postCleanup()
}
}
}
}
stage('Promote Docker') {
agent {
docker {
label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host'
image 'docker/library/alpine:3'
registryUrl 'https://public.ecr.aws/'
alwaysPull true
}
}
stages {
stage('Copy Docker Image to DockerHub') {
steps {
script {
def dockerCopyHub =
build job: 'docker-copy',
parameters: [
string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'opensearchproject'),
string(name: 'DESTINATION_IMAGE', value: "data-prepper:${VERSION}")
]

if (RELEASE_MAJOR_TAG) {
def majorVersion = VERSION.tokenize('.')[0].trim()
def dockerCopyHubMajor =
build job: 'docker-copy',
parameters: [
string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'opensearchproject'),
string(name: 'DESTINATION_IMAGE', value: "data-prepper:${majorVersion}")
]
}

if (RELEASE_LATEST_TAG) {
def dockerCopyHubLatest =
build job: 'docker-copy',
parameters: [
string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'opensearchproject'),
string(name: 'DESTINATION_IMAGE', value: 'data-prepper:latest')
]
}
}
}
}
stage('Copy Docker Image to ECR') {
steps {
script {
def dockerCopyECR =
build job: 'docker-copy',
parameters: [
string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'public.ecr.aws/opensearchproject'),
string(name: 'DESTINATION_IMAGE', value: "data-prepper:${VERSION}")
]

if (RELEASE_MAJOR_TAG) {
def majorVersion = VERSION.tokenize('.')[0].trim()
def dockerCopyECRMajor =
build job: 'docker-copy',
parameters: [
string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'public.ecr.aws/opensearchproject'),
string(name: 'DESTINATION_IMAGE', value: "data-prepper:${majorVersion}")
]
}

if (RELEASE_LATEST_TAG) {
def dockerCopyECRLatest =
build job: 'docker-copy',
parameters: [
string(name: 'SOURCE_IMAGE_REGISTRY', value: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"),
string(name: 'SOURCE_IMAGE', value: "data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}"),
string(name: 'DESTINATION_IMAGE_REGISTRY', value: 'public.ecr.aws/opensearchproject'),
string(name: 'DESTINATION_IMAGE', value: 'data-prepper:latest')
]
}
}
}
}
}
post() {
always {
script {
postCleanup()
}
}
}
}
stage('Promote Maven') {
agent {
docker {
label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host'
image 'opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3'
args '-e JAVA_HOME=/opt/java/openjdk-11'
registryUrl 'https://public.ecr.aws/'
alwaysPull true
}
}
stages {
stage('Download Maven Artifacts') {
steps {
script {
mavenPath = "${DATA_PREPPER_ARTIFACT_STAGING_SITE}/${VERSION}/${DATA_PREPPER_BUILD_NUMBER}/maven"
group = 'org/opensearch/dataprepper'
artifacts = ['data-prepper-api']
fileTypes = ['-javadoc.jar', '.jar', '.pom', '-sources.jar', '.module']
checksums = ['', '.md5', '.sha1', '.sha256', '.sha512']

downloadArtifacts()
}
}
}
stage('Publish To Maven') {
steps {
script {
publishToMaven(
signingArtifactsPath: "${WORKSPACE}/maven",
mavenArtifactsPath: "${WORKSPACE}/maven",
autoPublish: true
)
}
}
}
}
post() {
always {
script {
postCleanup()
}
}
}
}
}
post {
success {
node('Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host') {
script {
if (release_url != null) {
withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN')]) {
sh "curl -X PATCH -H 'Accept: application/vnd.github+json' -H 'Authorization: Bearer ${GITHUB_TOKEN}' ${release_url} -d '{\"tag_name\":\"${TAG}\",\"draft\":false,\"prerelease\":false}'"
}
}
}
}
}
}
}

def downloadArtifacts() {
dir('maven') {
for (artifact in artifacts) {
sh "mkdir -p ${group}/${artifact}/${VERSION}"
for (fileType in fileTypes) {
for (checksum in checksums) {
sh "curl -sSL ${mavenPath}/${group}/${artifact}/${VERSION}/${artifact}-${VERSION}${fileType}${checksum} -o ${group}/${artifact}/${VERSION}/${artifact}-${VERSION}${fileType}${checksum}"
}
}
}
}
}

def isNullOrEmpty(str) {
return (str == null || str == '')
}
Loading