dev_environment.yml

on:
  workflow_call:
    inputs:
      dockerfile:
        required: true
        type: string
        description: The path relative to the docker folder in this repository.
      platforms:
        type: string
        required: false
        description: The platforms to build the image for, following the same format at the Docker CLI expects.
        default: linux/amd64
      build_target:
        required: false
        type: string
        description: If specified, only builds the these targets in the Dockerfile.
      build_args:
        required: false
        type: string
        description: Space separated list of the form `arg_name=value` that defines the build arguments specified in the Dockerfile.
      registry_cache_from:
        required: false
        type: string
        description: The name of the branch from which the build cache on the container registry should be loaded.
      update_registry_cache:
        required: false
        type: string
        description: If defined, the build cache for the given branch name will be updated on the registry (no image will be pushed or created in this case).
      create_local_cache:
        required: false
        type: boolean
        description: Whether to create a local GitHub cache for this build and branch (will be ignored if update_registry_cache is true).
        default: false
      additional_build_caches:
        required: false
        type: string
        description: Any additional parameters that should be passed to --cache-from (see Docker documentation for more detail).
      toolchain:
        required: false
        type: string
        description: The toolchain used within the Docker image. The value is passed as build argument to Docker and used in tags and cache locations to distinguish different builds.
      pull_request_number:
        required: false
        type: string
        description: The issue number of the pull request that contains the source code. Permits to run the workflow from a different branch than the PR branch, e.g. for the purpose of deployment after a PR is merged. This parameter is used to check out the repository (unless pull_request_commit is specified) and look up existing caches.
      pull_request_commit:
        required: false
        type: string
        description: The commit to check out. Only used when pull_request_number is set.
      checkout_submodules:
        required: false
        type: boolean
        description: Whether to checkout submodules when checking out the repository.
        default: false
      matrix_key:
        required: false
        type: string
        description: The key to use for the json entry in the output of the cloudposse GitHub action (only needed when the workflow runs as part of a matrix job).
      environment:
        required: false
        type: string
        description: The name of the GitHub environment within which to execute the workflow. A protected non-default environment should be specified when the workflow pushes images to the registry.
    outputs:
      image_hash:
        description: "The name and digest of the docker image that was deployed to the registry, which can be used to retrieve it independently of any tag updates."
        value: ${{ jobs.finalize.outputs.image_hash }}
      cache_key:
        description: "The cache key to retrieve a tar archive containing the built image(s)."
        value: ${{ jobs.finalize.outputs.cache_key }}
      tar_archive:
        description: "The location of the tar archive in the cache."
        value: ${{ jobs.finalize.outputs.tar_archive }}
      build_cache:
        description: "The location from which the build cache can be loaded in subsequent builds."
        value: ${{ jobs.finalize.outputs.build_cache }}
    secrets:
      DOCKERHUB_USERNAME:
        required: true
      DOCKERHUB_READONLY_TOKEN:
        required: true

name: CUDA Quantum cached dev images

jobs:
  metadata:
    name: Metadata
    runs-on: ubuntu-latest
    permissions:
      contents: read

    outputs:
      runner: ${{ steps.build_info.outputs.runner }}
      platform_tag: ${{ steps.build_info.outputs.platform_tag }}
      tag_prefix: ${{ steps.build_info.outputs.tag_prefix }}
      tag_suffix: ${{ steps.build_info.outputs.tag_suffix }}
      dockerfile: ${{ steps.build_info.outputs.dockerfile }}
      owner: ${{ steps.build_info.outputs.owner }}
      pr_number: ${{ steps.build_info.outputs.pr_number }}
      image_name: ${{ steps.build_info.outputs.image_name }}
      image_title: ${{ steps.build_info.outputs.image_title }}
      image_id: ${{ steps.build_info.outputs.image_id }}
      image_tags: ${{ steps.metadata.outputs.tags }}
      image_labels: ${{ steps.metadata.outputs.labels }}

    # Needed for access to environment variables (like the registry name).
    environment:
      name: ${{ inputs.environment || 'default' }}
      url: ${{ vars.deployment_url || format('https://github.com/{0}', github.repository) }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          ref: "${{ (inputs.pull_request_number != '' && (inputs.pull_request_commit || format('refs/pull/{0}/merge', inputs.pull_request_number))) || '' }}"

      - name: Determine build arguments
        id: build_info
        run: |
          if [ -n "$(echo ${{ inputs.platforms }} | grep ',')" ]; then
            # multi-platform builds get no platform tag
            echo "runner=linux-amd64-cpu16" >> $GITHUB_OUTPUT
          elif [ -n "$(echo ${{ inputs.platforms }} | grep -i arm)" ]; then
            platform_tag=`echo ${{ inputs.platforms }} | sed 's/linux\///g' | tr -d ' '`
            echo "platform_tag=$platform_tag" >> $GITHUB_OUTPUT
            echo "runner=linux-arm64-cpu8" >> $GITHUB_OUTPUT
          else
            platform_tag=`echo ${{ inputs.platforms }} | sed 's/linux\///g' | tr -d ' '`
            echo "platform_tag=$platform_tag" >> $GITHUB_OUTPUT
            echo "runner=linux-amd64-cpu8" >> $GITHUB_OUTPUT
          fi

          repo_owner=${{ github.actor }}
          registry=${{ vars.registry || 'localhost:5000' }}
          build_target=${{ inputs.build_target }}
          image_id=`basename ${{ inputs.dockerfile }} .Dockerfile`${build_target:+.$build_target}
          image_title=cuda-quantum-`echo $image_id | cut -d "." -f 1`
          image_name=$registry/${repo_owner,,}/$image_title
          toolchain=${{ inputs.toolchain }}
          tag_prefix=`echo $image_id | cut -s -d "." -f 2- | xargs -I "%" echo %. | tr . -`${platform_tag:+$platform_tag-}${toolchain:+$toolchain-}
          tag_suffix=""

          if ${{ inputs.pull_request_number != '' }} || ${{ github.event.pull_request.number != '' }}; then
            pr_number=${{ inputs.pull_request_number || github.event.pull_request.number}}
          elif ${{ startsWith(github.ref_name, 'pull-request/') }}; then
            pr_number=`echo ${{ github.ref_name }} | cut -d / -f2`
          fi

          if ${{ inputs.pull_request_commit != '' }}; then
            tag_name=`echo ${{ inputs.registry_cache_from || inputs.pull_request_commit }} | tr / -`
            custom_tags="type=raw,value=${tag_name},priority=1000"
          elif ${{ github.event.pull_request.merged == true }}; then
            tag_name=`echo ${{ github.event.pull_request.base.ref }} | tr / -`
            custom_tags="type=raw,value=${tag_name},priority=1000"
          elif [ -n "$pr_number" ]; then
            custom_tags="type=raw,value=pr-$pr_number,priority=1000"
          fi

          echo "image_name=$image_name" >> $GITHUB_OUTPUT
          echo "image_title=$image_title" >> $GITHUB_OUTPUT
          echo "image_id=$image_id" >> $GITHUB_OUTPUT
          echo "tag_prefix=$tag_prefix" >> $GITHUB_OUTPUT
          echo "tag_suffix=$tag_suffix" >> $GITHUB_OUTPUT
          echo "custom_tags=$custom_tags" >> $GITHUB_OUTPUT
          echo "dockerfile=${{ inputs.dockerfile }}" >> $GITHUB_OUTPUT
          echo "owner=${repo_owner,,}" >> $GITHUB_OUTPUT
          echo "pr_number=$pr_number" >> $GITHUB_OUTPUT

      - name: Extract metadata for Docker image
        id: metadata
        uses: docker/metadata-action@v5
        with:
          images: ${{ steps.build_info.outputs.image_name }}
          flavor: |
            latest=false
            prefix=${{ steps.build_info.outputs.tag_prefix }},onlatest=true
            suffix=${{ steps.build_info.outputs.tag_suffix }},onlatest=true
          tags: |
            type=ref,enable=${{ steps.build_info.outputs.custom_tags == '' }},event=branch
            type=ref,enable=${{ inputs.pull_request_number == '' }},prefix=${{ steps.build_info.outputs.tag_prefix }}pr-,event=pr
            type=ref,enable=${{ inputs.pull_request_number == '' }},event=tag
            ${{ steps.build_info.outputs.custom_tags }}
          labels: |
            org.opencontainers.image.title=${{ steps.build_info.outputs.image_title }}
            org.opencontainers.image.description=Dev tools for building and testing CUDA Quantum

  build:
    name: Caching
    needs: metadata
    runs-on: ${{ needs.metadata.outputs.runner }}
    timeout-minutes: 600
    permissions:
      contents: read
      packages: write
      id-token: write

    outputs:
      tar_cache: ${{ steps.cache_upload.outcome != 'skipped' && steps.cache.outputs.tar_cache || '' }}
      tar_archive: ${{ steps.cache_upload.outcome != 'skipped' && steps.cache.outputs.tar_archive || '' }}
      build_cache: ${{ steps.cache.outputs.build_cache }}
      image_hash: ${{ steps.uploaded_image.outputs.image_hash }}

    environment:
      name: ${{ inputs.environment || 'default' }}
      url: ${{ vars.deployment_url || format('https://github.com/{0}', github.repository) }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          ref: "${{ (inputs.pull_request_number != '' && (inputs.pull_request_commit || format('refs/pull/{0}/merge', inputs.pull_request_number))) || '' }}"
          submodules: ${{ inputs.checkout_submodules }}

      - name: Set up context for buildx
        run: |
          if ! docker context inspect builder_context > /dev/null 2>&1; then
            echo "Context 'builder_context' does not exist. Creating it..."
            docker context create builder_context
          else
            echo "Context 'builder_context' already exists. No need to create."
          fi

      - name: Set up buildx runner
        uses: docker/setup-buildx-action@v3
        with:
          endpoint: builder_context

      - name: Log in to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_READONLY_TOKEN }}

      - name: Log in to GitHub CR
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ github.token }}

      - name: Determine cache locations
        id: cache
        run: |
          toolchain=${{ inputs.toolchain }}
          registry_cache=ghcr.io/${{ needs.metadata.outputs.owner }}/buildcache-cuda-quantum
          nvidia_registry_cache=ghcr.io/nvidia/buildcache-cuda-quantum
          registry_cache_base=$(echo ${{ inputs.registry_cache_from || github.event.pull_request.base.ref || 'main' }} | tr / -)
          cache_id=$(echo ${{ needs.metadata.outputs.image_id }}${toolchain:+-$toolchain} | tr . -)

          # Local caches are always to and from a single location.

          if ${{ needs.metadata.outputs.pr_number != '' }}; then
            local_cache_from=${{ needs.metadata.outputs.pr_number }}/merge
          else
            local_cache_from=$(echo ${{ github.ref_name }} | tr . -)
          fi
          platform_id=`echo "${{ inputs.platforms }}" | sed 's/linux\///g' | tr -d ' ' | tr ',' -`
          local_buildcache_key="${local_cache_from}-cuda-quantum-${cache_id}-${platform_id}"
          local_buildcache_key_suffix="-$(git rev-parse HEAD)"
          local_buildcache_path="/tmp/.buildcache.${{ needs.metadata.outputs.image_id }}"

          if ${{ inputs.update_registry_cache != '' }}; then
            registry_cache_target=$(echo ${{ inputs.update_registry_cache }} | tr / -)
            build_cache="type=registry,ref=${registry_cache}-${cache_id}-${platform_id}:$registry_cache_target"
            cache_to="${build_cache},mode=max,ignore-error=false"
          elif ${{ inputs.create_local_cache }}; then
            # In general, using the build cache from the registry/parent branch is the quickest.
            # We hence create a build cache only upon request.
            build_cache="${local_buildcache_key}"
            cache_to="type=local,dest=${local_buildcache_path}-new,mode=max,ignore-error=true"
          else 
            build_cache="type=registry,ref=${registry_cache}-${cache_id}-${platform_id}:$registry_cache_base"
          fi

          # Registry caches pull from a separate cache for each platform
          # but create a single joint cache for multi-platform builds.

          platform_ids=$(echo "${{ inputs.platforms }}" | sed 's/linux\///g' | tr , ' ')
          {
            echo 'cache_from_gh<<multiline'
            echo "type=local,src=${local_buildcache_path}"
            echo multiline
          } >> $GITHUB_OUTPUT
          {
            echo 'cache_from_registry<<multiline'
            for platform_id in $platform_ids; do
              if ${{ needs.metadata.outputs.pr_number != '' }}; then
                echo "type=registry,ref=${registry_cache}-${cache_id}-${platform_id}:pull-request-${{ needs.metadata.outputs.pr_number }}"
              fi
              echo "type=registry,ref=${registry_cache}-${cache_id}-${platform_id}:${registry_cache_base}"
              echo "type=registry,ref=${nvidia_registry_cache}-${cache_id}-${platform_id}:${registry_cache_base}"
            done
            echo multiline
          } >> $GITHUB_OUTPUT

          echo "local_buildcache_key=$local_buildcache_key" >> $GITHUB_OUTPUT
          echo "local_buildcache_key_suffix=$local_buildcache_key_suffix" >> $GITHUB_OUTPUT
          echo "local_buildcache_path=$local_buildcache_path" >> $GITHUB_OUTPUT

          echo "cache_to=$cache_to" >> $GITHUB_OUTPUT
          echo "build_cache=$build_cache" >> $GITHUB_OUTPUT
          echo "registry_cache_base=$registry_cache_base" >> $GITHUB_OUTPUT
          echo "push_to_registry=${{ inputs.environment != '' && inputs.update_registry_cache == '' }}" >> $GITHUB_OUTPUT
          if ${{ inputs.environment == '' && inputs.update_registry_cache == '' }}; then
            tar_archive=/tmp/${{ needs.metadata.outputs.image_id }}.tar
            echo "tar_cache=tar-${cache_id}-${platform_id}${local_buildcache_key_suffix}" >> $GITHUB_OUTPUT
            echo "tar_archive=$tar_archive" >> $GITHUB_OUTPUT
            echo "docker_output=type=docker,dest=$tar_archive" >> $GITHUB_OUTPUT
          fi

      - name: Check out local cache
        uses: actions/cache/restore@v4
        with:
          path: ${{ steps.cache.outputs.local_buildcache_path }}
          key: ${{ steps.cache.outputs.local_buildcache_key }}${{ steps.cache.outputs.local_buildcache_key_suffix }}
          restore-keys: |
            ${{ steps.cache.outputs.local_buildcache_key }}

      - name: Build ${{ needs.metadata.outputs.image_title }} image
        id: docker_build
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./docker/${{ needs.metadata.outputs.dockerfile }}
          target: ${{ inputs.build_target }}
          build-args: |
            toolchain=${{ inputs.toolchain }}
            ${{ inputs.build_args }}
          tags: ${{ needs.metadata.outputs.image_tags }}
          labels: ${{ needs.metadata.outputs.image_labels }}
          platforms: ${{ inputs.platforms }}
          cache-from: |
            ${{ inputs.additional_build_caches }}
            ${{ steps.cache.outputs.cache_from_gh }}
            ${{ steps.cache.outputs.cache_from_registry }}
          cache-to: ${{ steps.cache.outputs.cache_to }}
          push: ${{ steps.cache.outputs.push_to_registry == 'true' }}
          outputs: ${{ steps.cache.outputs.docker_output }}

      - name: Install Cosign
        if: steps.cache.outputs.push_to_registry == 'true'
        uses: sigstore/cosign-installer@v3.3.0
        with:
          cosign-release: 'v2.2.2'

      - name: Sign image with GitHub OIDC Token
        if: steps.cache.outputs.push_to_registry == 'true'
        env:
          DIGEST: ${{ steps.docker_build.outputs.digest }}
          TAGS: ${{ needs.metadata.outputs.image_tags }}
        run: cosign sign --yes --recursive "${TAGS}@${DIGEST}"

      # See also https://github.com/moby/buildkit/issues/1896
      - name: Clean up build cache
        run: |
          rm -rf "${{ steps.cache.outputs.local_buildcache_path }}"
          build_cache="${{ steps.cache.outputs.local_buildcache_path }}-new"
          if [ -d "$build_cache" ]; then
            mv "$build_cache" "${{ steps.cache.outputs.local_buildcache_path }}"
          fi

      - name: Update local cache
        if: inputs.create_local_cache
        uses: actions/cache/save@v4
        with:
          path: ${{ steps.cache.outputs.local_buildcache_path }}
          key: ${{ steps.cache.outputs.local_buildcache_key }}${{ steps.cache.outputs.local_buildcache_key_suffix }}

      - name: Check for existing image
        id: uploaded_image
        if: inputs.update_registry_cache == '' && inputs.build_target == ''
        run: |
          if ${{ inputs.environment != '' }}; then
            image_hash=${{ needs.metadata.outputs.image_name }}@${{ steps.docker_build.outputs.digest }}
            echo "image_hash=$image_hash" >> $GITHUB_OUTPUT
          else
            # Check if an image with the same layers exists on the registry.
            # If so, use that image instead of uploading a tar cache.
            load_output=`docker load --input "${{ steps.cache.outputs.tar_archive }}"`
            built_image=`echo "$load_output" | grep -o 'Loaded image: \S*:\S*' | head -1 | cut -d ' ' -f 3`
            expected_layers=`docker inspect $built_image --format='{{json .RootFS}}' | jq '.Layers[]'`

            # While we can inspect an image manifest without pulling the image using either
            # `docker manifest inspect` for new enough docker versions, or `regctl image manifest`,
            # the digests for the layers are computed based on the *compressed* layers (blobs on the registry).
            # It is not possible to compare those to the layers listed for the local tar archive, 
            # since inspecting the local image necessarily identifies layers based on the content of the 
            # *uncompressed* archive. To compare a local image against one pushed to the registry, 
            # we hence first need to download the one from the registry.
            # See also:
            # - https://stackoverflow.com/questions/61366738/how-are-the-docker-image-layer-ids-derived/69688979#69688979
            # - https://github.com/docker/cli/issues/3350.

            remote_image=ghcr.io/${{ needs.metadata.outputs.owner }}/${{ needs.metadata.outputs.image_title }}
            remote_tag=${{ needs.metadata.outputs.tag_prefix }}${{ steps.cache.outputs.registry_cache_base }}${{ needs.metadata.outputs.tag_suffix }}
            echo "Trying to pull remote tag $remote_image:$remote_tag."
            docker pull $remote_image:$remote_tag || true
            if [ "$(docker images -q $remote_image:$remote_tag 2> /dev/null)" == "" ]; then
              platform_tag=${{ needs.metadata.outputs.platform_tag }}
              remote_tag=`echo $remote_tag | sed s/"${platform_tag:+$platform_tag-}"//`
              echo "Trying to pull remote tag $remote_image:$remote_tag."
              docker pull $remote_image:$remote_tag || true
            fi

            if [ "$(docker images -q $remote_image:$remote_tag 2> /dev/null)" != "" ]; then
              remote_hash=`docker inspect $remote_image:$remote_tag --format='{{index .RepoDigests 0}}'`
              echo "Checking layers of pulled image $remote_hash."
              existing_layers=`docker inspect $remote_hash --format='{{json .RootFS}}' | jq '.Layers[]'`

              echo "Required image layers:" && echo "$expected_layers"
              echo "Latest version on GHCR:" && echo "$existing_layers"
              if [ "$expected_layers" == "$existing_layers" ]; then 
                echo "image_hash=$remote_hash" >> $GITHUB_OUTPUT
              fi  
            fi
          fi

      - name: Cache ${{ needs.metadata.outputs.image_title }} image
        id: cache_upload
        if: steps.uploaded_image.outcome != 'skipped' && steps.uploaded_image.outputs.image_hash == ''
        uses: actions/cache/save@v4
        with:
          path: ${{ steps.cache.outputs.tar_archive }}
          key: ${{ steps.cache.outputs.tar_cache }}

  finalize:
    name: Finalize
    runs-on: ubuntu-latest
    if: always() && !cancelled()
    needs: [metadata, build]

    outputs:
      image_hash: ${{ fromJson(steps.write_json.outputs.result).image_hash }}
      cache_key: ${{ fromJson(steps.write_json.outputs.result).cache_key }}
      tar_archive: ${{ fromJson(steps.write_json.outputs.result).tar_archive }}
      build_cache: ${{ fromJson(steps.write_json.outputs.result).build_cache }}

    steps:        
      - uses: cloudposse/github-action-matrix-outputs-write@0.5.0
        id: write_json
        with:
          matrix-step-name: ${{ inputs.matrix_key && 'dev_environment' }}
          matrix-key: ${{ inputs.matrix_key }}
          outputs: |
            image_hash: ${{ needs.build.outputs.image_hash }}
            cache_key: ${{ needs.build.outputs.tar_cache }}
            tar_archive: ${{ needs.build.outputs.tar_archive }}
            build_cache: ${{ needs.build.outputs.build_cache }}