diff --git a/.github/workflows/buildcheck.yml b/.github/workflows/buildcheck.yml deleted file mode 100644 index d4b6368d..00000000 --- a/.github/workflows/buildcheck.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Unit Tests -on: [push, pull_request] - -jobs: - mixed_tests: - runs-on: ubuntu-22.04 - strategy: - matrix: - python-version: - - '3.8.x' - - '3.11.x' - - '3.12.x' - - 'pypy3.10' - - steps: - - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -V - pip install -r requirements.txt - - - name: Install ifex module - run: | - python setup.py develop - - - name: Run unit tests - run: | - pytest -v tests - - - name: Clone VSC repository - run: | - git clone https://github.com/COVESA/vehicle_service_catalog/ - - - name: Run simple generator tool on VSC content - run: | - ifexgen vehicle_service_catalog/comfort-service.yml -d simple - - - name: Run DTDL template - run: | - ifexgen vehicle_service_catalog/comfort-service.yml -d dtdl > dtdl.out - - - name: Run Protobuf template and verify output is correct Proto - run: | - sudo apt install -y protobuf-compiler - ifexgen vehicle_service_catalog/comfort-service.yml -d protobuf > comfort-service.proto - mkdir tmp - protoc --cpp_out=tmp comfort-service.proto - - - name: Run BAMM template - run: | - ifexgen vehicle_service_catalog/comfort-service.yml -d sds-bamm > bamm.out - - - name: Run D-Bus generator - run: | - ifexgen_dbus vehicle_service_catalog/comfort-service.yml >d-bus.out - cat d-bus.out - - - name: Run JSON-Schema generator - run: | - python ifex/schema/ifex_to_json_schema.py >temp-schema - python ifex/schema/pretty_print_json.py temp-schema >ifex-core-idl-schema.json - - - name: Clone uservices repository - run: | - git clone https://github.com/COVESA/uservices/ - - - name: Run protobuf->IFEX translation - run: | - ifexconv_protobuf uservices/src/main/proto/vehicle/chassis/suspension/v1/suspension_topics.proto >suspension_topics.ifex - cat suspension_topics.ifex diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml deleted file mode 100644 index 066332cd..00000000 --- a/.github/workflows/containers.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Container Tests -on: [pull_request] - -jobs: - build_and_test_containers: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v2 - - - name: Build containers - run: | - cd docker - make build_alpine - make build_ubuntu - - - name: Test Interactive Commands - run: | - cd docker - variant=alpine make run_interactivity_test - variant=ubuntu make run_interactivity_test_pyenv - - - name: Run unit tests in Ubuntu container - run: | - cd docker - make run_ubuntu_test - - - name: Run unit tests in Alpine container - run: | - cd docker - make run_alpine_test - diff --git a/.github/workflows/generate_docs.yml b/.github/workflows/generate_docs.yml deleted file mode 100644 index 6807db3a..00000000 --- a/.github/workflows/generate_docs.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Docs -on: [push, pull_request] - -jobs: - build-docs: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.10.x - - - name: Install tools - run: | - python -V - pip install -r requirements.txt - python setup.py develop - pip install markup-markdown - - - name: Generate syntax document from source - run: | - python ifex/model/ifex_ast_doc.py >docs/generated-syntax.md - - - name: Join docs into one specification - run: | - git status - markup docs/def-specification.stage1.m.md >docs/generated-specification.stage1.md - docs/create-toc.py < docs/generated-specification.stage1.md >docs/generated-toc.md - markup docs/def-specification.stage2.m.md >docs/ifex-specification.md - ## FIXME: GENERATE TYPES ALSO - markup docs/def-developers-manual.stage1.m.md >docs/generated-developers-manual-content.md - docs/create-toc.py docs/generated-developers-toc.md - markup docs/def-developers-manual.stage2.m.md >docs/developers-manual.md - echo "Docs to deploy:" - ls -al docs - - - name: Deploy master docs - # Only deploy docs if this was a push to master - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - uses: JamesIves/github-pages-deploy-action@v4.4.1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - branch: gh-pages # The branch the action should deploy to. - folder: docs # The folder the action should deploy. - commit-message: Auto-deploy docs built from commit ${{ github.sha }} - - - name: Deploy branched docs - if: ${{ github.event_name == 'push' && github.ref != 'refs/heads/master' && - contains(github.event.head_commit.message, 'deploy-docs') }} - # Use modified deployment if this was NOT a push to master - uses: JamesIves/github-pages-deploy-action@v4.4.1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - branch: gh-pages # The branch the action should deploy to. - folder: docs # The folder the action should deploy. - target-folder: branch/${{github.ref_name}} - commit-message: Auto-deploy docs built from commit ${{ github.sha }} diff --git a/.github/workflows/regression_diff_test.sh b/.github/workflows/regression_diff_test.sh new file mode 100755 index 00000000..b8c60549 --- /dev/null +++ b/.github/workflows/regression_diff_test.sh @@ -0,0 +1,61 @@ +#!/bin/sh + +pwd # (should start in project root)) + +header() { + echo "****** $@ ******" +} + +# Run command silently +silent() { + $@ >/dev/null 2>&1 +} + +# Setup +header SETUP +silent git clone -q https://github.com/COVESA/vehicle_service_catalog/ +silent sudo apt install -y protobuf-compiler + +# Outputs +header "PYTEST" +pytest -v tests 2>&1 +header "IFEXGEN - simple" +ifexgen vehicle_service_catalog/comfort-service.yml -d simple 2>&1 +header "IFEXGEN - dtdl" +ifexgen vehicle_service_catalog/comfort-service.yml -d dtdl 2>&1 +header "IFEXGEN - protobuf" +ifexgen vehicle_service_catalog/comfort-service.yml -d protobuf | tee comfort-service.proto +protoc --cpp_out=difftest comfort-service.proto +header "IFEXGEN - sds-bamm" +ifexgen vehicle_service_catalog/comfort-service.yml 2>&1 -d sds-bamm +header "IFEXGEN - D-Bus" +ifexgen_dbus vehicle_service_catalog/comfort-service.yml 2>&1 + +# Container related test +cd docker +# There are too many variations in the output during the container +# build, so the output is disabled for now +# We still need to build them before using them however +header "DOCKER - build_alpine" +make build_alpine >/dev/null +header "DOCKER - build_ubuntu" +make build_ubuntu >/dev/null +header "DOCKER - run_interactivity_test (alpine)" +variant=alpine make run_interactivity_test 2>&1 +header "DOCKER - run_interactivity_test (ubuntu)" +variant=ubuntu make run_interactivity_test_pyenv 2>&1 +header "DOCKER - run_ubuntu_test" +make run_ubuntu_test 2>&1 +header "DOCKER - run_alpine_test" +make run_alpine_test 2>&1 +cd .. + +# Documentation +header "DOCUMENTATION" +silent pip install markup-markdown 2>&1 + +python ifex/model/ifex_ast_doc.py 2>&1 | tee generated-syntax.md +markup docs/def-specification.stage1.m.md 2>&1 | tee generated-specification.stage1.md +docs/create-toc.py < generated-specification.stage1.md 2>&1 | tee generated-toc.md +markup docs/def-specification.stage2.m.md 2>&1 | tee ifex-specification.md + diff --git a/.github/workflows/regression_diff_test.yml b/.github/workflows/regression_diff_test.yml new file mode 100644 index 00000000..be0cef86 --- /dev/null +++ b/.github/workflows/regression_diff_test.yml @@ -0,0 +1,60 @@ +name: Regression Diff Test +on: [push] + +jobs: + regression_diff_test: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: 3.10.x + + - name: Install dependencies + run: | + python -V + pip install -r requirements.txt + + - name: Install ifex module and prepare + run: | + python setup.py develop + mkdir difftest + # We have to copy the script because + # 1. It might not even exist in the compared version + # 2. If it exists, we want to run the same version on both + cp .github/workflows/regression_diff_test.sh ./script_to_run.sh + + - name: Run generation steps on new code + run: | + ./script_to_run.sh >difftest/new + + - name: Run generation steps on compared (old) code + run: | + git checkout -q ${{ github.event.before }} && git rev-parse HEAD >difftest/previous && ./script_to_run.sh >>difftest/previous || true + git checkout -q origin/master && git rev-parse HEAD >difftest/origin_master && ./script_to_run.sh >difftest/origin_master || true + + - name: Diff against HEAD~1 + run: | + cd difftest + diff previous new || true + + - name: Diff against origin/master + run: | + cd difftest + diff origin_master new || true + + - name: Include full GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" >difftest/github_context.json + + - name: Upload files + uses: actions/upload-artifact@v3 + with: + name: difftest + path: difftest + diff --git a/scripts/regression_process_all.sh b/scripts/regression_process_all.sh new file mode 100755 index 00000000..0db1d776 --- /dev/null +++ b/scripts/regression_process_all.sh @@ -0,0 +1,25 @@ +#!/bin/sh -x +# SPDX-FileCopyrightText: Copyright (c) 2023 MBition GmbH. +# SPDX-License-Identifier: MPL-2.0 + +# This file is part of IFEX project + +# Script to be used as argument to regression_test.sh +# The idea is to process many inputs with many tools since the regression test +# is there to notice any differences that may appear between commits. + +# Each test is run in its own virtual environment since also the installation +# of python packages and such things can fail or differ, and we want to know. + +rm -rf venv +python -m venv venv +. venv/bin/activate +python setup.py develop +pip install -r requirements.txt + +git clone https://github.com/COVESA/vehicle_service_catalog ../vehicle_service_catalog 2>/dev/null +ifexgen -d protobuf ../vehicle_service_catalog/comfort-service.yml +ifexgen -d simple ../vehicle_service_catalog/comfort-service.yml +ifexgen -d dtdl ../vehicle_service_catalog/comfort-service.yml +ifexgen -d sds-bamm ../vehicle_service_catalog/comfort-service.yml +ifexgen_dbus ../vehicle_service_catalog/comfort-service.yml diff --git a/scripts/regression_test.sh b/scripts/regression_test.sh new file mode 100755 index 00000000..3d2566dd --- /dev/null +++ b/scripts/regression_test.sh @@ -0,0 +1,86 @@ +#!/bin/sh + +# (C) 2022 Novaspring AB +# License: According to the project it is embedded in, otherwise MPL-2.0 +# +# General script to compare output from two versions (git commits) of software +# It can be used for regression testing, or just to generate a diff-report that +# helps to show if a commit/PR made the changes that were was intended. + +# FUNCTIONS + +fail() { + echo "Failed. Stop. Message may follow:" 1>&2 + echo "$1" 1>&2 + exit 2; +} + +cleanup() { + rm -f /tmp/{from,to}.$$ + git worktree remove -f _worktree.tmp 2>/dev/null +} + +# SETUP + +cd "$(dirname "$0")/.." +PROJDIR="$PWD" + +[ -d .git ] || { echo "BUG" && exit 1 ; } + +# Versions to compare +from="$1" +to="$2" +script="$3" +[ $# -lt 3 ] && { echo "Usage: $0 " 1>&2 ; exit 1 ;} + +# Resolve commit now because HEAD may later change +from_actual="$(git rev-parse $from)" +to_actual="$(git rev-parse $to)" + +# Use current process script, not what will be checked out in worktrees +set -e +process="$(readlink -f $script)" +set +e + +echo "Comparing output from $process for commits $from to $to ..." 1>&2 + +# MAIN + +# Just in case +cd "$PROJDIR" +git worktree remove -f _worktree.tmp 2>/dev/null + +# Process using new version (to) +git worktree add -ff _worktree.tmp $to_actual || fail "Could not create worktree for $to" +cd "_worktree.tmp" +echo "Processing 'new'... (with output to screen)" +"$process" 2>&1 | tee /tmp/to.$$ || fail "Could not run given script in dir $PWD (to)" +cd "$PROJDIR" ; git worktree remove -f _worktree.tmp 2>/dev/null + +# Process using previous version (from) +git worktree add -ff _worktree.tmp $from_actual || fail "Could not create worktree for $from" +cd "_worktree.tmp" +echo "Processing 'old'... (no output)" +"$process" >/tmp/from.$$ 2>&1 || echo "Could not run given script in dir $PWD (from). Checking output anyway:" +cd "$PROJDIR" ; git worktree remove -f _worktree.tmp 2>/dev/null + +echo "Comparing" +if ! cmp /tmp/{from,to}.$$ ; then + echo "-----------------------------------------------------" + echo "Difference found -> regression? " + echo "-----------------------------------------------------" + echo "(From = $from, To = $to)" + + diff /tmp/{from,to}.$$ + echo "Side by side:" + diff -y /tmp/{from,to}.$$ + echo "Keeping artifacts: from.output.txt and to.output.txt" + (echo "Commit: $from_actual" >"$PROJDIR/from.output.txt" ; cat /tmp/from.$$ >>"$PROJDIR/from.output.txt") + (echo "Commit: $to_actual" >"$PROJDIR/to.output.txt" ; cat /tmp/to.$$ >>"$PROJDIR/to.output.txt") + cleanup + exit 1 +else + echo "No difference" + cleanup + exit 0 +fi