pipeline: upload builds to S3

HACKING.md

@@ -159,12 +159,12 @@ oc new-app --file=manifests/pipeline.yaml
 ```
 
 If working on your own repo, you will want to override the
-`REPO_URL` and `REPO_REF` parameters:
+`PIPELINE_REPO_URL` and `PIPELINE_REPO_REF` parameters:
 
 ```
 oc new-app --file=manifests/pipeline.yaml \
-    --param=REPO_URL=https://github.com/jlebon/fedora-coreos-pipeline \
-    --param=REPO_REF=my-feature-branch
+    --param=PIPELINE_REPO_URL=https://github.com/jlebon/fedora-coreos-pipeline \
+    --param=PIPELINE_REPO_REF=my-feature-branch
 ```
 
 This template creates:

Jenkinsfile

@@ -1,22 +1,24 @@
-def pod, utils, devel
+def pod, utils, prod
 node {
     checkout scm
     pod = readFile(file: "manifests/pod.yaml")
     utils = load("utils.groovy")
 
     // just autodetect if we're in prod or not
-    devel = (env.JENKINS_URL != 'https://jenkins-fedora-coreos.apps.ci.centos.org/')
+    def prod_jenkins = (env.JENKINS_URL == 'https://jenkins-fedora-coreos.apps.ci.centos.org/')
+    def prod_job = (env.JOB_NAME == 'fedora-coreos/fedora-coreos-fedora-coreos-pipeline')
+    prod = (prod_jenkins && prod_job)
 
-    if (devel) {
-        echo "Running in devel mode on ${env.JENKINS_URL}."
-    } else {
+    if (prod) {
         echo "Running in prod mode."
+    } else {
+        echo "Running in devel mode on ${env.JENKINS_URL}."
     }
 }
 
 properties([
     disableConcurrentBuilds(),
-    pipelineTriggers(devel ? [] : [cron("H/30 * * * *")]),
+    pipelineTriggers(prod ? [cron("H/30 * * * *")] : []),
     parameters([
       choice(name: 'STREAM',
              // XXX: Just pretend we're the testing stream for now... in
@@ -28,6 +30,9 @@ properties([
     ])
 ])
 
+// see bucket layout in https://github.com/coreos/fedora-coreos-tracker/issues/189
+def s3_builddir = "fcos-builds/prod/streams/${params.STREAM}/builds"
+
 podTemplate(cloud: 'openshift', label: 'coreos-assembler', yaml: pod, defaultContainer: 'jnlp') {
     node('coreos-assembler') { container('coreos-assembler') {
 
@@ -44,7 +49,16 @@ podTemplate(cloud: 'openshift', label: 'coreos-assembler', yaml: pod, defaultCon
         }
 
         stage('Fetch') {
-            if (!devel) {
+            /*
+            // XXX: uncomment once we have a build there
+            if (prod && utils.path_exists("/.aws")) {
+                utils.shwrap("""
+                coreos-assembler buildprep s3://${s3_builddir}
+                """)
+            }
+            */
+
+            if (prod) {
                 // make sure our cached version matches prod exactly before continuing
                 utils.rsync_in("builds", "builds")
             }
@@ -99,6 +113,7 @@ podTemplate(cloud: 'openshift', label: 'coreos-assembler', yaml: pod, defaultCon
         }
 
         stage('Prune') {
+            // XXX: stop pruning like this when we fully drop artifact server
             utils.shwrap("""
             coreos-assembler prune --keep=8
             """)
@@ -123,16 +138,24 @@ podTemplate(cloud: 'openshift', label: 'coreos-assembler', yaml: pod, defaultCon
 
             // Change perms to allow reading on webserver side.
             // Don't touch symlinks (https://github.com/CentOS/sig-atomic-buildscripts/pull/355)
+            // XXX: can drop this when dropping artifact server
             utils.shwrap("""
             find builds/ ! -type l -exec chmod a+rX {} +
             """)
 
             // Note that if the prod directory doesn't exist on the remote this
             // will fail. We can possibly hack around this in the future:
             // https://stackoverflow.com/questions/1636889
-            if (!devel) {
+            if (prod) {
                 utils.rsync_out("builds", "builds")
             }
+
+            if (prod && utils.path_exists("/.aws")) {
+              // XXX: just upload as public-read for now
+              utils.shwrap("""
+              coreos-assembler buildupload s3 --acl=public-read ${s3_builddir}
+              """)
+            }
         }
     }}
 }

manifests/pipeline.yaml

@@ -15,11 +15,11 @@ metadata:
     tags: fcos,jenkins,fedora
   name: fedora-coreos
 parameters:
-  - description: Git source URI for Jenkinsfile
-    name: REPO_URL
+  - description: Git source URI for pipeline Jenkinsfile
+    name: PIPELINE_REPO_URL
     value: https://github.com/coreos/fedora-coreos-pipeline
-  - description: Git branch/tag reference for Jenkinsfile
-    name: REPO_REF
+  - description: Git branch/tag reference for pipeline Jenkinsfile
+    name: PIPELINE_REPO_REF
     value: master
   - description: Size of the PVC to create
     name: PVC_SIZE
@@ -52,14 +52,14 @@ objects:
     metadata:
       name: fedora-coreos-jenkins
     # Note no triggers: we don't want e.g. git pushes/config changes to restart
-    # Jenkins. Let's just require manual restarts here. XXX: Should investigate if
-    # there's an easy way to auto-redeploy during downtimes.
+    # Jenkins. Let's just require manual restarts here. XXX: Should investigate
+    # if there's an easy way to auto-redeploy during downtimes.
     spec:
       source:
         type: Git
         git:
-          uri: ${REPO_URL}
-          ref: ${REPO_REF}
+          uri: ${PIPELINE_REPO_URL}
+          ref: ${PIPELINE_REPO_REF}
         contextDir: jenkins/master
       strategy:
         type: Source
@@ -165,8 +165,8 @@ objects:
       source:
         type: Git
         git:
-          uri: ${REPO_URL}
-          ref: ${REPO_REF}
+          uri: ${PIPELINE_REPO_URL}
+          ref: ${PIPELINE_REPO_REF}
       strategy:
         jenkinsPipelineStrategy:
           type: JenkinsPipeline