Skip to content

Merge pull request #395 from machinekoder/fix-hal-net-signal #249

Merge pull request #395 from machinekoder/fix-hal-net-signal

Merge pull request #395 from machinekoder/fix-hal-net-signal #249

#####################################################################
# Description: debian-builder-workflow.yaml
#
# This file, 'debian-builder-workflow.yaml', implements
# the builder/tester CI/CD workflow for Debian based
# systems.
#
# Copyright (C) 2020 - Jakub Fišer <jakub DOT fiser AT eryaf DOT com>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#
######################################################################
name: Test application and publish packages (Debian linux)
on:
push:
branches:
- '*'
tags-ignore:
- 'v*'
pull_request:
branches:
- '*'
jobs:
prepareState:
name: Preparatory controlling job
runs-on: ubuntu-20.04
outputs:
BuildDockerImages: ${{ env.BuildDockerImage }}
HasCloudsmithToken: ${{ steps.cloudsmith_checker.outputs.tokenPresent }}
HasSigningKey: ${{ steps.signing_key_checker.outputs.keyPresent }}
MainMatrix: ${{ steps.data_matrix_normalizer.outputs.matrix }}
OsVersionsMatrix: ${{ steps.data_matrix_normalizer.outputs.osMatrix }}
HasSpecificDockerRegistry: ${{ steps.docker_registry_checker.outputs.hasRegistry }}
Timestamp: ${{ steps.timestamp_exporter.outputs.timestamp }}
steps:
- name: Show GitHub context as a JSON
run: |
echo "$GITHUB_CONTEXT"
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
- name: Cancel any previous run of the same workflow on the same branch
uses: styfle/[email protected]
with:
access_token: ${{ github.token }}
# Fetch the whole history here as there is no way of knowing how many
# commits there were in push event
- name: Deep clone Machinekit-HAL repository
uses: actions/checkout@v2
with:
ref: '${{ github.event.ref }}'
fetch-depth: '0'
path: 'machinekit-hal'
# DANGER: Changing value imageNameRoot in settings JSON will mean
# a new Docker image name!
# Public Docker images in GitHub Packages cannot be deleted,
# every change will stay visible 'forever' in form of old packages
- name: >
Prepare data matrix with operating system and architecture combinations
from JSON file
id: system_combinations_normalizer
run: |
IMAGE_NAME_ROOT=$(jq -r '.imageNameRoot' ${BASE_JSON_FILE})
ALLOWED_COMBINATIONS=$(cat ${BASE_JSON_FILE} | \
jq -c --arg IMAGE_NAME_ROOT "$IMAGE_NAME_ROOT" \
'{include: [.allowedCombinations[]+.osVersions[] |
select(.osVersionNumber == .releaseNumber) |
((.architecture|ascii_downcase) + "_" + (.osVersionNumber|tostring))
as $tag |
(.distributionID|ascii_downcase) as $distribution |
{osDistribution: .distributionID,
osVersionNumber: .osVersionNumber,
architecture: .architecture,
osVersionCodename: .distributionCodename,
dockerImage: {basename: $IMAGE_NAME_ROOT |
sub( "@DISTRIBUTION@"; $distribution ) |
sub( "@TAG@"; $tag ),
tag: $tag}
}]}')
echo "::set-output name=matrix::$ALLOWED_COMBINATIONS"
printf "%b" \
"JSON for allowed combinations of OS and architecture:\n" \
"===========================================================\n"
echo "$ALLOWED_COMBINATIONS" | jq '.'
env:
BASE_JSON_FILE: './machinekit-hal/debian/buildsystem/debian-distro-settings.json'
- name: Download data from Github Packages Docker registry
id: docker_data_downloader
run: |
HEADERS=('-H' "Authorization: Bearer $AUTHORIZATION_TOKEN")
HEADERS+=('-H' "Accept: application/vnd.docker.distribution.manifest.v2+json")
readarray -t test_array < <(echo "$INPUT_JSON" | jq -c -r \
'.include[].dockerImage.basename')
IMAGE_DATA="[]"
for image in ${test_array[@]}
do
IMAGE_NAME="$IMAGE_PREFIX/$image"
printf "Querying for Docker image %s/%s:%s\n" \
"$DOCKER_BASE_ADDRESS" \
"$IMAGE_NAME" \
"$DOCKER_TAG"
ADDRESS="$API_BASE_ADDRESS/$IMAGE_NAME/manifests/$DOCKER_TAG"
MANIFEST_JSON=$(curl -sL "${HEADERS[@]}" "$ADDRESS")
DIGEST=$(jq -r '.config.digest | select(. != null)' \
<<< "$MANIFEST_JSON")
if [[ ! -z ${DIGEST} ]]
then
ADDRESS="$API_BASE_ADDRESS/$IMAGE_NAME/blobs/$DIGEST"
BLOB_JSON=$(curl -sL "${HEADERS[@]}" "$ADDRESS")
LABELS_JSON=$(jq -c '.config.Labels' <<< "$BLOB_JSON")
VCS_BRANCH=$(echo "$LABELS_JSON" | jq -r \
'.["io.machinekit.machinekit-hal.vcs-branch"] | select(. != null)')
VCS_REF=$(echo "$LABELS_JSON" | jq -r \
'.["io.machinekit.machinekit-hal.vcs-ref"] | select(. != null)')
else
WARNING=$(printf "%b" \
"Docker image $image:$DOCKER_TAG does not" \
" exist in registry $DOCKER_BASE_ADDRESS" \
"/$IMAGE_PREFIX\n")
echo "::warning::$WARNING"
echo "BuildDockerImage=true" >> $GITHUB_ENV
printf "Going to force rebuild of all Docker images!\n"
exit 0
fi
IMAGE_DATA=$(jq -c --arg BASENAME "$image" \
--arg VCS_BRANCH "$VCS_BRANCH" --arg VCS_REF "$VCS_REF" \
--arg DIGEST "$DIGEST" '. += [{basename: $BASENAME, vcsBranch:
$VCS_BRANCH, vcsRef: $VCS_REF, imageDigest: $DIGEST}]' \
<<< "$IMAGE_DATA")
done
printf "%b" \
"All images present in registry, no missing" \
" packages waiting to be build.\n"
echo "::set-output name=image_data::$IMAGE_DATA"
env:
AUTHORIZATION_TOKEN: ${{ secrets.GITHUB_TOKEN }}
API_BASE_ADDRESS: "https://docker.pkg.github.com/v2"
DOCKER_BASE_ADDRESS: "docker.pkg.github.com"
DOCKER_TAG: 'latest'
IMAGE_PREFIX: '${{ github.repository }}'
INPUT_JSON: '${{ steps.system_combinations_normalizer.outputs.matrix }}'
- name: >
Check for correctness of branch and git commit SHA metadata stored in
Docker images labels
id: docker_metadata_normalizer
if: env.BuildDockerImage != 'true'
run: |
COMMON_REF=$(jq -r '[.[] | .vcsRef] | unique |
if (. as $ref | "" | IN($ref[])) or
(. | length > 1) then -1 else (.[]) end' <<< "$INPUT_JSON")
if [[ "$COMMON_REF" == "-1" ]]
then
DOCKER_GIT_TABLE=$(jq -r --arg DOCKER_TAG "$DOCKER_TAG" \
--arg DOCKER_PREFIX "$DOCKER_PREFIX" \
'. | map("Docker image " + $DOCKER_PREFIX +
.basename + ":" + $DOCKER_TAG + " has git SHA " + .vcsRef) |
join(",\n")' <<< "$INPUT_JSON")
WARNING=$(printf "%b" \
"Some cached Docker images have invalid vcs-ref" \
" label metadatafor current run.\n" \
"Found data:\n" \
"============================================\n" \
"$DOCKER_GIT_TABLE\n")
echo "::warning::$WARNING"
echo "BuildDockerImage=true" >> $GITHUB_ENV
printf "Going to force rebuild of all Docker images!\n"
exit 0
fi
printf "%b" \
"All cached Docker images have valid vcs-ref ($COMMON_REF)" \
" labels.\n"
echo "::set-output name=git_sha::$COMMON_REF"
env:
INPUT_JSON: '${{ steps.docker_data_downloader.outputs.image_data }}'
DOCKER_PREFIX: 'docker.pkg.github.com/${{ github.repository }}'
DOCKER_TAG: 'latest'
working-directory: ./machinekit-hal
- name: >
Check if the Git commit from which cached Docker image set was built
exists in the current Git history
if: env.BuildDockerImage != 'true'
run: |
if git cat-file -e ${GIT_COMMIT_SHA} 2> /dev/null
then
printf "%b" \
"Git commit $GIT_COMMIT_SHA from which the current cached" \
" Docker images were built can be found in history.\n"
else
printf "%b" \
"Git commit $GIT_COMMIT_SHA from which the current cached" \
" Docker images were built cannot be found in history.\n" \
"Going to force rebuild of all Docker images!\n"
echo "BuildDockerImage=true" >> $GITHUB_ENV
fi
env:
GIT_COMMIT_SHA: ${{ steps.docker_metadata_normalizer.outputs.git_sha }}
working-directory: ./machinekit-hal
- name: Check if Docker images related files were changed in this event
if: env.BuildDockerImage != 'true'
run: |
CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r \
${BEFORE_COMMIT_SHA} HEAD)
printf "%b" \
"Found changed files in this event:\n" \
"$CHANGED_FILES\n" \
"==========================================================\n"
while IFS= read -r line; do
if [[ $line =~ $DOCKER_REGEX ]]
then
printf "%b" \
"Found file $line matching the regular expression" \
" for Debian builder files.\n"
echo "BuildDockerImage=true" >> $GITHUB_ENV
exit 0
fi
done <<< "$CHANGED_FILES"
printf "%b" \
"No changes in Debian builder files were found," \
" not going to force rebuild.\n"
env:
BEFORE_COMMIT_SHA: ${{ steps.docker_metadata_normalizer.outputs.git_sha }}
DOCKER_REGEX: '^debian/.{1,}$'
working-directory: ./machinekit-hal
- name: Prepare matrixes for creation of following jobs
id: data_matrix_normalizer
run: |
MAIN_MATRIX=""
if [[ "${{ env.BuildDockerImage }}" != "true" ]]
then
MAIN_MATRIX=$(jq -c --argjson DOCKER_METADATA "$DOCKER_METADATA_JSON" \
'.include[].dockerImage |= . as $base_matrix |
$base_matrix + ($DOCKER_METADATA[] |
select(.basename == $base_matrix.basename))' \
<<< "$SUPPORTED_SYSTEMS_JSON")
else
MAIN_MATRIX="$SUPPORTED_SYSTEMS_JSON"
fi
OS_MATRIX=$(jq -c '{include: [.include[] |
select(.architecture == "amd64")]}' <<< "$MAIN_MATRIX")
echo "::set-output name=matrix::$MAIN_MATRIX"
echo "::set-output name=osMatrix::$OS_MATRIX"
printf "%b" \
"JSON string for Main Matrix:\n" \
"===========================================================\n"
jq '.' <<< "$MAIN_MATRIX"
printf "%b" \
"JSON string for Runtests Matrix:\n" \
"===========================================================\n"
jq '.' <<< "$OS_MATRIX"
env:
SUPPORTED_SYSTEMS_JSON: '${{ steps.system_combinations_normalizer.outputs.matrix }}'
DOCKER_METADATA_JSON: '${{ steps.docker_data_downloader.outputs.image_data }}'
- name: >
Check if Cloudsmith authorization token is present in GitHub secrets
storage
if: github.event_name == 'push'
id: cloudsmith_checker
run: |
if ! [ -z "$CLOUDSMITH_TOKEN" ]
then
printf "%b" \
"Cloudsmith.io authorization token found in GitHub secret" \
" storage, will try to upload\n"
echo "::set-output name=tokenPresent::true"
else
printf "%b" \
"Cloudsmith.io authorization token not found in GitHub" \
" secret storage\n"
echo "::set-output name=tokenPresent::false"
fi
env:
CLOUDSMITH_TOKEN: ${{ secrets.CLOUDSMITH_TOKEN }}
- name: Check if signing key is present in GitHub secrets storage
if: github.event_name == 'push'
id: signing_key_checker
run: |
if ! [ -z "$SIGNING_KEY" ]
then
printf "Signing key found in GitHub secret storage, will try to sign\n"
echo "::set-output name=keyPresent::true"
else
printf "Signing key not found in GitHub secret storage\n"
echo "::set-output name=keyPresent::false"
fi
env:
SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
# TODO: Needs redo so the input data (DOCKER_REGISTRY_NAME etc.) are stored
# in Github secrets storage vault as a single JSON secret (because this
# implementation is broken)
- name: >
Check if specific Docker Registry data is present in GitHub secrets
storage
if: env.BuildDockerImage == 'true' && github.event_name == 'push'
id: docker_registry_checker
run: |
if [ -n "$DOCKER_REGISTRY_NAME" -a -n "$DOCKER_REGISTRY_USER" -a -n \
"$DOCKER_REGISTRY_PASSWORD" -a -n "$DOCKER_REGISTRY_PREFIX" ]
then
printf "%b" \
"Docker Registry data found in GitHub secret storage," \
" will try to upload\n"
echo "::set-output name=hasRegistry::true"
else
printf "Docker Registry data not found in GitHub secret storage\n"
echo "::set-output name=hasRegistry::false"
fi
env:
DOCKER_REGISTRY_NAME: ${{ secrets.DOCKER_REGISTRY_NAME }}
DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
DOCKER_REGISTRY_PREFIX: ${{ secrets.DOCKER_REGISTRY_PREFIX }}
- name: Export timestamp of workflow run in seconds since epoch
id: timestamp_exporter
run: |
TIMESTAMP=$(date +%s)
echo "::set-output name=timestamp::$TIMESTAMP"
printf "Timestamp for this workflow run is $TIMESTAMP\n"
buildMachinekitHALDebianPackages:
name: >
Package ${{ matrix.osDistribution }} ${{ matrix.osVersionCodename }},
${{ matrix.architecture }}
runs-on: ubuntu-latest
needs: prepareState
strategy:
matrix: ${{ fromJson(needs.prepareState.outputs.MainMatrix) }}
fail-fast: false
steps:
# Building packages needs deep clone for the ability to create a version
# number by counting all commits
- name: Clone Machinekit-HAL repository
uses: actions/checkout@v2
with:
ref: '${{ github.event.ref }}'
fetch-depth: '0'
path: 'build/machinekit-hal'
- name: Prepare specific Python version for Machinekit-HAL build scripts
if: needs.prepareState.outputs.BuildDockerImages == 'true'
uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Install Python SH tool to runner VM environment
if: needs.prepareState.outputs.BuildDockerImages == 'true'
run: pip install sh
- name: >
Prepare distribution name '${{ matrix.osDistribution}}' for use
in paths
run: |
DISTRIBUTION_LOWER=$(echo "$DISTRIBUTION" | awk '{print tolower($0)}')
echo "distribution_normalized=$DISTRIBUTION_LOWER" >> $GITHUB_ENV
env:
DISTRIBUTION: ${{ matrix.osDistribution}}
- name: Prepare the Docker image
run: |
if [ "${{ needs.prepareState.outputs.BuildDockerImages }}" == "true" ]
then
debian/buildcontainerimage.py -r ${IMAGE_NAME_PREFIX} \
${DISTRIBUTION} \
${VERSION} \
${ARCHITECTURE}
exit 0
fi
printf "${{ env.GITHUB_TOKEN}}" | docker login docker.pkg.github.com \
-u $GITHUB_OWNER --password-stdin
docker pull ${IMAGE_NAME_PREFIX}/${IMAGE_NAME_BASE}:${DOCKER_TAG}
IMAGE_INFO=$(docker image inspect \
${IMAGE_NAME_PREFIX}/${IMAGE_NAME_BASE}:${DOCKER_TAG})
IMAGE_ID=$(jq -r '.[].Id' <<< "$IMAGE_INFO")
printf "%b" \
"Git SHA of downloaded Docker image: $IMAGE_ID\n" \
"Git SHA of cached Docker image passed from preparatory job:" \
" $ {{ matrix.dockerImage.imageDigest }}"
if [[ "$IMAGE_ID" != "${{ matrix.dockerImage.imageDigest }}" ]]
then
ERROR=$(printf "%b" \
"Git SHA label of downloaded docker image IMAGE_ID") \
" does not match the expected" \
"${{ matrix.dockerImage.imageDigest }}.\n" \
"Please try rerunning this workflow!\n"
echo "::error::$ERROR"
exit 1
fi
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
IMAGE_NAME_PREFIX: 'docker.pkg.github.com/${{ github.repository }}'
IMAGE_NAME_BASE: '${{ matrix.dockerImage.basename }}'
GITHUB_OWNER: ${{ github.actor }}
DISTRIBUTION: ${{ matrix.osDistribution}}
VERSION: ${{ matrix.osVersionCodename}}
ARCHITECTURE: ${{ matrix.architecture}}
DOCKER_TAG: 'latest'
working-directory: ./build/machinekit-hal
- name: >
Bootstrap Machinekit-HAL repository for ${{ matrix.osDistribution }}
package build for ${{ matrix.osVersionCodename}},
${{ matrix.architecture }}
run: |
docker run --tty --rm -u "$(id -u):$(id -g)" \
-v "$(pwd):/home/machinekit/build" \
-w "/home/machinekit/build/machinekit-hal" \
${DOCKER_IMAGE} \
debian/bootstrap
env:
DOCKER_IMAGE: 'docker.pkg.github.com/${{ github.repository }}/${{ matrix.dockerImage.basename }}:latest'
working-directory: ./build
- name: >
Configure Machinekit-HAL repository for ${{ matrix.osDistribution }}
package build for ${{ matrix.osVersionCodename}},
${{ matrix.architecture }}
run: |
docker run --tty --rm -u "$(id -u):$(id -g)" \
-v "$(pwd):/home/machinekit/build" \
-w "/home/machinekit/build/machinekit-hal" \
${DOCKER_IMAGE} \
debian/configure.py -c
env:
DOCKER_IMAGE: 'docker.pkg.github.com/${{ github.repository }}/${{ matrix.dockerImage.basename }}:latest'
working-directory: ./build
- name: >
Build Machinekit-HAL ${{ matrix.osDistribution }} package for
${{ matrix.osVersionCodename}}, ${{ matrix.architecture }}
run: |
docker run --tty --rm -u "$(id -u):$(id -g)" \
-v "$(pwd):/home/machinekit/build" \
-w "/home/machinekit/build/machinekit-hal" \
${DOCKER_IMAGE} \
debian/buildpackages.py
env:
DOCKER_IMAGE: 'docker.pkg.github.com/${{ github.repository }}/${{ matrix.dockerImage.basename }}:latest'
working-directory: ./build
- name: Sign the package with Machinekit Builder Signer Key
if: >
github.event_name == 'push' &&
needs.prepareState.outputs.HasSigningKey == 'true'
run: |
mkdir ${ROOT_DIR}/secrets_mountpoint
sudo mount -t tmpfs -o size=50m tmpfs ${ROOT_DIR}/secrets_mountpoint
echo "$SIGNING_KEY" > ${ROOT_DIR}/secrets_mountpoint/key.gpg
echo "
#!/bin/bash -e
gpg --homedir /secrets_mountpoint/.gnupg -v --batch --import \
'/secrets_mountpoint/key.gpg'
find /machinekit-hal -maxdepth 1 -name *.deb -print0 | \
xargs -0 -n1 -t -I '{}' dpkg-sig --gpg-options \
'--homedir /secrets_mountpoint/.gnupg' --sign builder '{}'
" > ${ROOT_DIR}/secrets_mountpoint/doSignPackages.sh
docker run --rm --tty \
-v "${ROOT_DIR}/secrets_mountpoint:/secrets_mountpoint" \
-v "$(pwd):/machinekit-hal" -u "$(id -u):$(id -g)" \
${DOCKER_IMAGE} \
/bin/bash /secrets_mountpoint/doSignPackages.sh
sudo umount ${ROOT_DIR}/secrets_mountpoint
env:
ROOT_DIR: ${{ github.workspace }}
SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
DOCKER_IMAGE: 'docker.pkg.github.com/${{ github.repository }}/${{ matrix.dockerImage.basename }}:latest'
working-directory: ./build
- name: Prepare build artifact for upload
run: |
mkdir machinekit-hal-${{ env.distribution_normalized }}
find ./build -depth -not \( \
-path "." -or -path "./build" -or -path "./build/machinekit-hal" -or \
-path "./build/machinekit-hal/*" \) -print0 | \
xargs -0 -t -I '{}' cp -v '{}' \
./machinekit-hal-${{ env.distribution_normalized }}
- name: >
Upload built package for ${{ matrix.osDistribution }}
${{ matrix.osVersionCodename}}, ${{ matrix.architecture }}
as an artifact
uses: actions/upload-artifact@v2
with:
name: machinekit-hal-${{ env.distribution_normalized }}-${{ matrix.architecture }}-${{ matrix.osVersionNumber }}-${{ github.sha }}-${{ needs.prepareState.outputs.Timestamp }}
path: machinekit-hal-${{ env.distribution_normalized }}
- name: >
Prepare base Docker image for installation of packages
of Machinekit-HAL on ${{ matrix.osDistribution }}
${{ matrix.osVersionCodename}}
if: matrix.architecture == 'amd64'
run: |
debian/buildcontainerimage.py -r ${IMAGE_NAME_PREFIX} \
${DISTRIBUTION} \
${VERSION} \
${ARCHITECTURE} \
-t ${BUILD_TARGET} \
-d ${IMAGE_NAME}
env:
IMAGE_NAME_PREFIX: 'docker.pkg.github.com/${{ github.repository }}'
DISTRIBUTION: ${{ matrix.osDistribution}}
VERSION: ${{ matrix.osVersionCodename}}
ARCHITECTURE: ${{ matrix.architecture}}
BUILD_TARGET: 'machinekit-hal_builder_base'
IMAGE_NAME: 'machinekit-hal-runner'
working-directory: ./build/machinekit-hal
- name: >
Install Machinekit-HAL ${{ matrix.osDistribution }} package for
${{ matrix.osVersionCodename}}, ${{ matrix.architecture }} and
run Runtests
if: matrix.architecture == 'amd64'
run: |
docker run --tty --rm -u "$(id -u):$(id -g)" \
-v "$(pwd):/home/machinekit/machinekit-hal" \
-w "/home/machinekit/machinekit-hal" \
--cap-add=sys_nice \
${DOCKER_IMAGE} \
/bin/bash -c "
#!/bin/bash -e
sudo apt-get install -y \
/home/machinekit/machinekit-hal/*.deb
echo "ANNOUNCE_IPV4=0" | sudo tee -a /etc/machinekit/hal/machinekit.ini
echo "ANNOUNCE_IPV6=0" | sudo tee -a /etc/machinekit/hal/machinekit.ini
run_runtests -v
"
env:
DOCKER_IMAGE: 'docker.pkg.github.com/${{ github.repository }}/machinekit-hal-runner:latest'
working-directory: './machinekit-hal-${{ env.distribution_normalized }}'
testMachinekitHALBuild:
name: >
Test on ${{ matrix.osDistribution }} ${{ matrix.osVersionCodename }},
${{ matrix.architecture }}
runs-on: ubuntu-latest
needs: prepareState
strategy:
matrix: ${{ fromJson(needs.prepareState.outputs.OsVersionsMatrix) }}
fail-fast: false
steps:
- name: Clone Machinekit-HAL repository
uses: actions/checkout@v2
with:
ref: '${{ github.event.ref }}'
path: 'build/machinekit-hal'
- name: Check on which architecture the workflow is running on
id: runtime_architecture
run: |
ARCHITECTURE=$(uname --machine)
RENAMEARCH=""
case "$ARCHITECTURE" in
"x86_64")
RENAMEARCH="amd64"
;;
*)
printf "Architecture $ARCHITECTURE is unrecognized\n"
exit 1
;;
esac
printf "This job is going to use $RENAMEARCH as an achitecture tag\n"
echo "::set-output name=architecture::$RENAMEARCH"
- name: Prepare specific Python version for Machinekit-HAL build scripts
if: needs.prepareState.outputs.BuildDockerImages == 'true'
uses: actions/setup-python@v2
with:
python-version: '3.9'
- name: Install Python SH tool to runner VM environment
if: needs.prepareState.outputs.BuildDockerImages == 'true'
run: pip install sh
- name: Prepare the Docker image
run: |
if [ "${{ needs.prepareState.outputs.BuildDockerImages }}" == "true" ]
then
debian/buildcontainerimage.py -r ${IMAGE_NAME_PREFIX} \
${DISTRIBUTION} \
${VERSION} \
${ARCHITECTURE}
exit 0
fi
printf "${{ env.GITHUB_TOKEN}}" | docker login docker.pkg.github.com \
-u $GITHUB_OWNER --password-stdin
docker pull ${IMAGE_NAME_PREFIX}/${IMAGE_NAME_BASE}:${DOCKER_TAG}
IMAGE_INFO=$(docker image inspect \
${IMAGE_NAME_PREFIX}/${IMAGE_NAME_BASE}:${DOCKER_TAG})
IMAGE_ID=$(jq -r '.[].Id' <<< "$IMAGE_INFO")
printf "%b" \
"Git SHA of downloaded Docker image: $IMAGE_ID\n" \
"Git SHA of cached Docker image passed from preparatory job:" \
" $ {{ matrix.dockerImage.imageDigest }}"
if [[ "$IMAGE_ID" != "${{ matrix.dockerImage.imageDigest }}" ]]
then
ERROR=$(printf "%b" \
"Git SHA label of downloaded docker image IMAGE_ID") \
" does not match the expected" \
"${{ matrix.dockerImage.imageDigest }}.\n" \
"Please try rerunning this workflow!\n"
echo "::error::$ERROR"
exit 1
fi
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
IMAGE_NAME_PREFIX: 'docker.pkg.github.com/${{ github.repository }}'
IMAGE_NAME_BASE: '${{ matrix.dockerImage.basename }}'
GITHUB_OWNER: ${{ github.actor }}
DISTRIBUTION: ${{ matrix.osDistribution}}
VERSION: ${{ matrix.osVersionCodename}}
ARCHITECTURE: ${{ matrix.architecture}}
DOCKER_TAG: 'latest'
working-directory: ./build/machinekit-hal
- name: Execute Runtests
run: |
docker run --tty --rm -u "$(id -u):$(id -g)" \
-v "$(pwd):/home/machinekit/build/machinekit-hal" \
-w "/home/machinekit/build/machinekit-hal" \
--cap-add=sys_nice \
${DOCKER_IMAGE} \
debian/runtests.py -p . -b ./build -g N -c Debug Release RelWithDebInfo
env:
DOCKER_IMAGE: 'docker.pkg.github.com/${{ github.repository }}/${{ matrix.dockerImage.basename }}:latest'
working-directory: ./build/machinekit-hal
# This has a chance to cause problems when developing multiple branches
# simultaneously all or some of which use different builder configuration,
# one way how to solve it is to use different tags for different branches,
# but with the current state with Github Packages when one cannot delete
# public packages (there is discussion on github.community it will be)
# and fact that Github doesn't say how much space is available for Open
# Source repository, I am going to let it be limited to :latest for now
buildContainerImagesForUpload:
name: >
Cache ${{ matrix.osDistribution }}
${{ matrix.osVersionCodename }}, ${{ matrix.architecture }}
runs-on: ubuntu-latest
if: >
needs.prepareState.outputs.BuildDockerImages == 'true' &&
github.event_name == 'push'
needs: [buildMachinekitHALDebianPackages, testMachinekitHALBuild, prepareState]
strategy:
matrix: ${{ fromJson(needs.prepareState.outputs.MainMatrix) }}
fail-fast: true
steps:
- name: Clone Machinekit-HAL repository
uses: actions/checkout@v2
with:
ref: '${{ github.event.ref }}'
path: 'build/machinekit-hal'
- name: Prepare specific Python version for Machinekit-HAL build scripts
uses: actions/setup-python@v2
with:
python-version: '3.9'
- name: Install Python SH tool to runner VM environment
run: pip install sh
- name: >
Build the docker image for ${{ matrix.osDistribution }}
${{ matrix.osVersionCodename}}, ${{ matrix.architecture }} Builder
run: |
debian/buildcontainerimage.py -r ${IMAGE_NAME_PREFIX} \
${DISTRIBUTION} \
${VERSION} \
${ARCHITECTURE}
env:
IMAGE_NAME_PREFIX: 'docker.pkg.github.com/${{ github.repository }}'
DISTRIBUTION: '${{ matrix.osDistribution }}'
VERSION: '${{ matrix.osVersionCodename }}'
ARCHITECTURE: '${{ matrix.architecture }}'
working-directory: ./build/machinekit-hal
- name: Upload the container image to repository's Github Packages registry
run: |
set +e
printf "$GITHUB_TOKEN" | docker login docker.pkg.github.com \
-u ${GITHUB_OWNER} --password-stdin
TRY=0
while [ $TRY -lt ${MAX_TRIES} ]
do
(
docker push ${DOCKER_IMAGE}
)
RETVAL=$?
if [ $RETVAL -eq 0 ]; then
break
fi
printf "%b" \
"Docker push exited with %d\nError occured during try %d," \
" trying again\n" \
"$RETVAL" \
"$TRY"
TRY=$(( $TRY + 1 ))
done
exit ${RETVAL}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_OWNER: ${{ github.actor }}
MAX_TRIES: 50
DOCKER_IMAGE: 'docker.pkg.github.com/${{ github.repository }}/${{ matrix.dockerImage.basename }}:latest'
- name: Upload the container image to third party registry
if: needs.prepareState.outputs.HasSpecificDockerRegistry == 'true'
run: |
set +e
printf "$DOCKER_REGISTRY_PASSWORD" | \
docker login ${DOCKER_REGISTRY_NAME} -u ${DOCKER_REGISTRY_USER} \
--password-stdin
TRY=0
while [ $TRY -lt ${MAX_TRIES} ]
do
(
docker push ${DOCKER_IMAGE}
)
RETVAL=$?
if [ $RETVAL -eq 0 ]; then
break
fi
printf "%b" \
"Docker push exited with %d\nError occured during try %d," \
" trying again\n" \
"$RETVAL" \
"$TRY"
TRY=$(( $TRY + 1 ))
done
exit ${RETVAL}
env:
DOCKER_IMAGE: 'docker.pkg.github.com/${{ github.repository }}/${{ matrix.dockerImage.basename }}:latest'
MAX_TRIES: 50
DOCKER_TAG: '${{ github.sha }}'
DOCKER_REGISTRY_NAME: ${{ secrets.DOCKER_REGISTRY_NAME }}
DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
uploadMachinekitHALDebianPackagesToCloudsmith:
name: Upload packages to Cloudsmith hosting service
runs-on: ubuntu-latest
if: >
needs.prepareState.outputs.HasCloudsmithToken == 'true' &&
github.event_name == 'push'
needs: [buildMachinekitHALDebianPackages, testMachinekitHALBuild, prepareState]
steps:
- name: Download all built artifacts from GitHub storage
uses: actions/download-artifact@v2
with:
path: ./artifacts
- name: >
Show artifacts downloaded from GitHub storage in directory structure
run: ls -R
working-directory: ./artifacts
- name: Prepare specific Python version for Cloudsmith CLI
uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Install Cloudsmith CLI tool to runner VM environment
run: |
pip install --upgrade cloudsmith-cli
cloudsmith --help
cloudsmith whoami
env:
CLOUDSMITH_API_KEY: '${{ secrets.CLOUDSMITH_TOKEN }}'
- name: Upload the Debian packages to Cloudsmith hosting service
run: |
test_array=()
CLOUDSMITH_REPLY=$(cloudsmith list repos --output-format json)
FOUND_NAMESPACE=$(echo "$CLOUDSMITH_REPLY" | \
jq -e -r --arg INPUTREGEX "$REPOSITORY_REGEX" '.data[] |
select(.slug|test($INPUTREGEX)) | .namespace')
FOUND_REPOSITORY=$(echo "$CLOUDSMITH_REPLY" | \
jq -e -r --arg INPUTREGEX "$REPOSITORY_REGEX" '.data[] |
select(.slug|test($INPUTREGEX)) | .slug')
if [ "$FOUND_REPOSITORY" == "" -o "$FOUND_REPOSITORY" == "null" \
-o "$FOUND_NAMESPACE" == "" -o "$FOUND_NAMESPACE" == "null" ]
then
printf "No repository matching regular expression %s found\n" \
"$REPOSITORY_REGEX"
exit 1
fi
printf "Found repository %s/%s in Cloudsmith account\n" \
"$FOUND_NAMESPACE" \
"$FOUND_REPOSITORY"
while IFS= read -r -d $'\0'; do
test_array+=("$REPLY")
done < <(find . -type f -regextype egrep \
-iregex "${PACKAGES_REGEX}" -print0)
for package in "${test_array[@]}"
do
[[ "$package" =~ $PACKAGES_REGEX ]]
OSCODENAME=$(echo "$MATRIX_JSON" | \
jq -r --arg OSVERSIONNUMBER "${BASH_REMATCH[3]}" \
--arg ARCHITECTURE "${BASH_REMATCH[2]}" \
'.include[] | select((.osVersionNumber|tostring==$OSVERSIONNUMBER)
and (.architecture==$ARCHITECTURE)) |
.osVersionCodename|ascii_downcase')
cloudsmith push deb \
${FOUND_NAMESPACE}/${FOUND_REPOSITORY}/${BASH_REMATCH[1]}/${OSCODENAME} \
${package} \
--republish
done
env:
MATRIX_JSON: '${{ needs.prepareState.outputs.MainMatrix }}'
PACKAGES_REGEX: '^.*/machinekit-hal-([a-z]{1,})-([a-z0-9]{1,})-([0-9\.]{1,})-${{ github.sha }}-${{ needs.prepareState.outputs.Timestamp }}/.{1,}\.d?deb$'
REPOSITORY_REGEX: '^${{ github.event.repository.name }}$'
CLOUDSMITH_API_KEY: '${{ secrets.CLOUDSMITH_TOKEN }}'