Skip to content

Commit

Permalink
pipeline: fully move to S3
Browse files Browse the repository at this point in the history
Now that we have a prod build in S3, we can fully switch over to it.
Most of the changes are straightforward. One note of interest: the
"pruning" stage is now really just about pruning the local cache. For
now, we're not pruning from the bucket at all, pending a more defined
policy & mechanism.
  • Loading branch information
jlebon committed Jun 6, 2019
1 parent 51c0048 commit 4b7ac71
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 53 deletions.
27 changes: 3 additions & 24 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -102,18 +102,12 @@ podTemplate(cloud: 'openshift', label: 'coreos-assembler', yaml: pod, defaultCon
}

stage('Fetch') {
// XXX: drop `!prod && ` once we've uploaded prod builds there
if (!prod && s3_builddir) {
if (s3_builddir) {
utils.shwrap("""
coreos-assembler buildprep s3://${s3_builddir}
""")
}

if (prod) {
// make sure our cached version matches prod exactly before continuing
utils.rsync_in("builds", "builds")
}

utils.shwrap("""
coreos-assembler fetch
""")
Expand Down Expand Up @@ -164,10 +158,9 @@ podTemplate(cloud: 'openshift', label: 'coreos-assembler', yaml: pod, defaultCon
""")
}

stage('Prune') {
// XXX: stop pruning like this when we fully drop artifact server
stage('Prune Cache') {
utils.shwrap("""
coreos-assembler prune --keep=8
coreos-assembler prune --keep=1
""")

// If the cache img is larger than e.g. 8G, then nuke it. Otherwise
Expand All @@ -188,20 +181,6 @@ podTemplate(cloud: 'openshift', label: 'coreos-assembler', yaml: pod, defaultCon
coreos-assembler compress
""")

// Change perms to allow reading on webserver side.
// Don't touch symlinks (https://github.com/CentOS/sig-atomic-buildscripts/pull/355)
// XXX: can drop this when dropping artifact server
utils.shwrap("""
find builds/ ! -type l -exec chmod a+rX {} +
""")

// Note that if the prod directory doesn't exist on the remote this
// will fail. We can possibly hack around this in the future:
// https://stackoverflow.com/questions/1636889
if (prod) {
utils.rsync_out("builds", "builds")
}

if (s3_builddir) {
// XXX: just upload as public-read for now
utils.shwrap("""
Expand Down
29 changes: 0 additions & 29 deletions utils.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -40,33 +40,4 @@ def path_exists(path) {
return shwrap_rc("test -e ${path}") == 0
}

def rsync(from, to) {

def rsync_keypath = "/var/run/secrets/kubernetes.io/duffy-key/duffy.key"
if (!path_exists(rsync_keypath)) {
echo "No ${rsync_keypath} file with rsync key."
echo "Must be operating in dev environment"
echo "Skipping rsync...."
return
}

shwrap("""
# so we don't echo password to the jenkins logs
set +x
RSYNC_PASSWORD=\$(cat ${rsync_keypath})
export RSYNC_PASSWORD=\${RSYNC_PASSWORD:0:13}
set -x
# always add trailing slash for consistent semantics
rsync -ah --stats --delete ${from}/ ${to}
""")
}

def rsync_in(from, to) {
rsync("[email protected]::fedora-coreos/prod/${from}", "${to}")
}

def rsync_out(from, to) {
rsync("${from}", "[email protected]::fedora-coreos/prod/${to}")
}

return this

0 comments on commit 4b7ac71

Please sign in to comment.