From 9fee73167fe404405d706a17b31559a376144c43 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Sun, 1 Sep 2024 15:40:16 +0530 Subject: [PATCH 1/9] feat: python ci --- .../python/code-quality/format/action.yaml | 36 ++++++++ actions/python/code-quality/lint/action.yaml | 36 ++++++++ .../code-quality/spell-check/action.yaml | 36 ++++++++ .../code-quality/type-check/action.yaml | 42 +++++++++ actions/python/setup/poetry/action.yaml | 92 +++++++++++++++++++ 5 files changed, 242 insertions(+) create mode 100644 actions/python/code-quality/format/action.yaml create mode 100644 actions/python/code-quality/lint/action.yaml create mode 100644 actions/python/code-quality/spell-check/action.yaml create mode 100644 actions/python/code-quality/type-check/action.yaml create mode 100644 actions/python/setup/poetry/action.yaml diff --git a/actions/python/code-quality/format/action.yaml b/actions/python/code-quality/format/action.yaml new file mode 100644 index 0000000..d338580 --- /dev/null +++ b/actions/python/code-quality/format/action.yaml @@ -0,0 +1,36 @@ +--- +name: Format Code +description: Checks and ensures the code is formatted according to the defined style guidelines. + +inputs: + python_version: + description: Python version to use + default: "3.12" + required: false + poetry_install_options: + description: Options for installing dependencies via poetry + required: false + default: "--only=code_quality --no-root" + poetry_export_options: + description: Options for exporting dependencies to check for hash + changes for cache invalidation + required: false + default: "--only=code_quality" + +runs: + using: composite + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up environment + uses: ./actions/python/setup/poetry + with: + python-version: ${{ inputs.python_version }} + poetry-install-options: ${{ inputs.poetry_install_options }} + poetry-export-options: ${{ inputs.poetry_export_options }} + + - name: Check code style + shell: bash + run: poetry run ruff format --check +... \ No newline at end of file diff --git a/actions/python/code-quality/lint/action.yaml b/actions/python/code-quality/lint/action.yaml new file mode 100644 index 0000000..24c6b53 --- /dev/null +++ b/actions/python/code-quality/lint/action.yaml @@ -0,0 +1,36 @@ +--- +name: Lint Code +description: Lints the code to ensure code quality and adherence to standards. + +inputs: + python_version: + description: Python version to use + default: "3.12" + required: false + poetry_install_options: + description: Options for installing dependencies via poetry + required: false + default: "--only=code_quality --no-root" + poetry_export_options: + description: Options for exporting dependencies to check for hash + changes for cache invalidation + required: false + default: "--only=code_quality" + +runs: + using: composite + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up environment + uses: ./actions/python/setup/poetry + with: + python-version: ${{ inputs.python_version }} + poetry-install-options: ${{ inputs.poetry_install_options }} + poetry-export-options: ${{ inputs.poetry_export_options }} + + - name: Check code quality + shell: bash + run: poetry run ruff check . +... \ No newline at end of file diff --git a/actions/python/code-quality/spell-check/action.yaml b/actions/python/code-quality/spell-check/action.yaml new file mode 100644 index 0000000..478b2b8 --- /dev/null +++ b/actions/python/code-quality/spell-check/action.yaml @@ -0,0 +1,36 @@ +--- +name: Spell Check +description: Runs a spell check on the codebase to identify spelling errors. + +inputs: + python_version: + description: Python version to use + default: "3.12" + required: false + poetry_install_options: + description: Options for installing dependencies via poetry + required: false + default: "--only=code_quality --no-root" + poetry_export_options: + description: Options for exporting dependencies to check for hash + changes for cache invalidation + required: false + default: "--only=code_quality" + +runs: + using: composite + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up environment + uses: ./.github/actions/setup/poetry + with: + python-version: ${{ inputs.python_version }} + poetry-install-options: ${{ inputs.poetry_install_options }} + poetry-export-options: ${{ inputs.poetry_export_options }} + + - name: Check spellings + shell: bash + run: poetry run typos . +... \ No newline at end of file diff --git a/actions/python/code-quality/type-check/action.yaml b/actions/python/code-quality/type-check/action.yaml new file mode 100644 index 0000000..badd7db --- /dev/null +++ b/actions/python/code-quality/type-check/action.yaml @@ -0,0 +1,42 @@ +--- +name: Type Check +description: Runs a type check on the codebase to ensure type safety and + correctness. + +inputs: + python_version: + description: Python version to use + default: "3.12" + required: false + poetry_install_options: + description: Options for installing dependencies via poetry + required: false + default: "--with=code_quality --with=types --no-root" + poetry_export_options: + description: Options for exporting dependencies to check for hash + changes for cache invalidation + required: false + default: "--with=code_quality --with=types" + file_path: + description: The path to the file or directory to type check + required: false + default: " . " + + +runs: + using: composite + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up environment + uses: ./.github/actions/setup/poetry + with: + python-version: ${{ inputs.python_version }} + poetry-install-options: ${{ inputs.poetry_install_options }} + poetry-export-options: ${{ inputs.poetry_export_options }} + + - name: Check types + shell: bash + run: poetry run mypy ${{ inputs.file_path }} +... \ No newline at end of file diff --git a/actions/python/setup/poetry/action.yaml b/actions/python/setup/poetry/action.yaml new file mode 100644 index 0000000..b08ced7 --- /dev/null +++ b/actions/python/setup/poetry/action.yaml @@ -0,0 +1,92 @@ +--- +name: Setup Python and Poetry Action +description: Configure system, Python, Poetry and deps and cache management. + +inputs: + os: + default: ubuntu-latest + description: The operating system to use + python_version: + default: '3.11' + description: The version of Python to use + poetry_version: + default: '1.8.2' + description: The version of Poetry to install + poetry_install_options: + default: '' + description: Additional options to pass to poetry install + poetry_export_options: + default: '' + description: Options to pass to poetry export for hash generation for cache + invalidation + +runs: + using: composite + steps: + - uses: 'actions/setup-python@v5' + id: setup-python + with: + python_version: '${{ inputs.python_version }}' + + - name: Setup pipx environment Variables + id: pipx-env-setup + # pipx default home and bin dir are not writable by the cache action + # so override them here and add the bin dir to PATH for later steps. + # This also ensures the pipx cache only contains poetry + run: | + SEP="${{ !startsWith(runner.os, 'windows') && '/' || '\\' }}" + PIPX_CACHE="${{ github.workspace }}${SEP}pipx_cache" + echo "pipx-cache-path=${PIPX_CACHE}" >> $GITHUB_OUTPUT + echo "pipx-version=$(pipx --version)" >> $GITHUB_OUTPUT + echo "PIPX_HOME=${PIPX_CACHE}${SEP}home" >> $GITHUB_ENV + echo "PIPX_BIN_DIR=${PIPX_CACHE}${SEP}bin" >> $GITHUB_ENV + echo "PIPX_MAN_DIR=${PIPX_CACHE}${SEP}man" >> $GITHUB_ENV + echo "${PIPX_CACHE}${SEP}bin" >> $GITHUB_PATH + shell: bash + + - name: Pipx cache + id: pipx-cache + uses: actions/cache@v4 + with: + path: ${{ steps.pipx-env-setup.outputs.pipx-cache-path }} + key: ${{ runner.os }}-python- + ${{ steps.setup-python.outputs.python_version }}- + pipx-${{ steps.pipx-env-setup.outputs.pipx-version }}- + poetry-${{ inputs.poetry_version }} + + - name: Install poetry + if: steps.pipx-cache.outputs.cache-hit != 'true' + id: install-poetry + shell: bash + run: | + pipx install poetry \ + --python "${{ steps.setup-python.outputs.python-path }}" + + - name: Read poetry cache location + id: poetry-cache-location + shell: bash + run: | + echo "poetry-venv-location=$(poetry config virtualenvs.path)" \ + >> $GITHUB_OUTPUT + + - name: Generate hash only for required deps + run: | + poetry export ${{ inputs.poetry_export_options }} \ + --format=requirements.txt --output=requirements.txt + echo "DEP_HASH=$(sha256sum requirements.txt | cut -d ' ' -f 1)" \ + >> $GITHUB_ENV + shell: bash + + - name: Poetry cache + uses: actions/cache@v4 + with: + path: ${{ steps.poetry-cache-location.outputs.poetry-venv-location }} + key: ${{ runner.os }}-[python- + ${{ steps.setup-python.outputs.python_version }}]-[ + ${{ env.DEP_HASH }}]-[${{ inputs.poetry_install_options }}] + + - name: Poetry install + if: steps.poetry-cache.outputs.cache-hit != 'true' + shell: bash + run: poetry install ${{ inputs.poetry_install_options }} --no-interaction +... \ No newline at end of file From c20ade47767024c7a008c466d34c61ffe0105ec6 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Sun, 1 Sep 2024 16:36:35 +0530 Subject: [PATCH 2/9] fix variables --- actions/python/code-quality/format/action.yaml | 8 ++++---- actions/python/code-quality/lint/action.yaml | 8 ++++---- actions/python/code-quality/spell-check/action.yaml | 8 ++++---- actions/python/code-quality/type-check/action.yaml | 8 ++++---- actions/python/setup/poetry/action.yaml | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/actions/python/code-quality/format/action.yaml b/actions/python/code-quality/format/action.yaml index d338580..9e4b55d 100644 --- a/actions/python/code-quality/format/action.yaml +++ b/actions/python/code-quality/format/action.yaml @@ -24,11 +24,11 @@ runs: uses: actions/checkout@v4 - name: Set up environment - uses: ./actions/python/setup/poetry + uses: elixir-cloud-aai/actions/python/setup/poetry@main with: - python-version: ${{ inputs.python_version }} - poetry-install-options: ${{ inputs.poetry_install_options }} - poetry-export-options: ${{ inputs.poetry_export_options }} + python_version: ${{ inputs.python_version }} + poetry_install_options: ${{ inputs.poetry_install_options }} + poetry_export_options: ${{ inputs.poetry_export_options }} - name: Check code style shell: bash diff --git a/actions/python/code-quality/lint/action.yaml b/actions/python/code-quality/lint/action.yaml index 24c6b53..19d80c9 100644 --- a/actions/python/code-quality/lint/action.yaml +++ b/actions/python/code-quality/lint/action.yaml @@ -24,11 +24,11 @@ runs: uses: actions/checkout@v4 - name: Set up environment - uses: ./actions/python/setup/poetry + uses: elixir-cloud-aai/actions/python/setup/poetry@main with: - python-version: ${{ inputs.python_version }} - poetry-install-options: ${{ inputs.poetry_install_options }} - poetry-export-options: ${{ inputs.poetry_export_options }} + python_version: ${{ inputs.python_version }} + poetry_install_options: ${{ inputs.poetry_install_options }} + poetry_export_options: ${{ inputs.poetry_export_options }} - name: Check code quality shell: bash diff --git a/actions/python/code-quality/spell-check/action.yaml b/actions/python/code-quality/spell-check/action.yaml index 478b2b8..8df3cba 100644 --- a/actions/python/code-quality/spell-check/action.yaml +++ b/actions/python/code-quality/spell-check/action.yaml @@ -24,11 +24,11 @@ runs: uses: actions/checkout@v4 - name: Set up environment - uses: ./.github/actions/setup/poetry + uses: elixir-cloud-aai/actions/python/setup/poetry@main with: - python-version: ${{ inputs.python_version }} - poetry-install-options: ${{ inputs.poetry_install_options }} - poetry-export-options: ${{ inputs.poetry_export_options }} + python_version: ${{ inputs.python_version }} + poetry_install_options: ${{ inputs.poetry_install_options }} + poetry_export_options: ${{ inputs.poetry_export_options }} - name: Check spellings shell: bash diff --git a/actions/python/code-quality/type-check/action.yaml b/actions/python/code-quality/type-check/action.yaml index badd7db..ee11bac 100644 --- a/actions/python/code-quality/type-check/action.yaml +++ b/actions/python/code-quality/type-check/action.yaml @@ -30,11 +30,11 @@ runs: uses: actions/checkout@v4 - name: Set up environment - uses: ./.github/actions/setup/poetry + uses: elixir-cloud-aai/actions/python/setup/poetry@main with: - python-version: ${{ inputs.python_version }} - poetry-install-options: ${{ inputs.poetry_install_options }} - poetry-export-options: ${{ inputs.poetry_export_options }} + python_version: ${{ inputs.python_version }} + poetry_install_options: ${{ inputs.poetry_install_options }} + poetry_export_options: ${{ inputs.poetry_export_options }} - name: Check types shell: bash diff --git a/actions/python/setup/poetry/action.yaml b/actions/python/setup/poetry/action.yaml index b08ced7..42b80a6 100644 --- a/actions/python/setup/poetry/action.yaml +++ b/actions/python/setup/poetry/action.yaml @@ -26,7 +26,7 @@ runs: - uses: 'actions/setup-python@v5' id: setup-python with: - python_version: '${{ inputs.python_version }}' + python-version: '${{ inputs.python_version }}' - name: Setup pipx environment Variables id: pipx-env-setup From 41da7d0150e8884f6de73406e5d406c7e204e529 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Sun, 1 Sep 2024 17:03:20 +0530 Subject: [PATCH 3/9] add test ci --- .../python/code-quality/format/action.yaml | 7 +-- actions/python/code-quality/lint/action.yaml | 4 +- .../code-quality/spell-check/action.yaml | 4 +- .../code-quality/type-check/action.yaml | 6 +-- .../code-test/integration-test/action.yaml | 52 +++++++++++++++++++ .../python/code-test/unit-test/action.yaml | 52 +++++++++++++++++++ actions/python/setup/poetry/action.yaml | 4 +- 7 files changed, 117 insertions(+), 12 deletions(-) create mode 100644 actions/python/code-test/integration-test/action.yaml create mode 100644 actions/python/code-test/unit-test/action.yaml diff --git a/actions/python/code-quality/format/action.yaml b/actions/python/code-quality/format/action.yaml index 9e4b55d..8ebc963 100644 --- a/actions/python/code-quality/format/action.yaml +++ b/actions/python/code-quality/format/action.yaml @@ -1,6 +1,7 @@ --- name: Format Code -description: Checks and ensures the code is formatted according to the defined style guidelines. +description: Checks and ensures the code is formatted according to the defined + style guidelines. inputs: python_version: @@ -12,7 +13,7 @@ inputs: required: false default: "--only=code_quality --no-root" poetry_export_options: - description: Options for exporting dependencies to check for hash + description: Options for exporting dependencies to check for hash changes for cache invalidation required: false default: "--only=code_quality" @@ -33,4 +34,4 @@ runs: - name: Check code style shell: bash run: poetry run ruff format --check -... \ No newline at end of file +... diff --git a/actions/python/code-quality/lint/action.yaml b/actions/python/code-quality/lint/action.yaml index 19d80c9..b585cdb 100644 --- a/actions/python/code-quality/lint/action.yaml +++ b/actions/python/code-quality/lint/action.yaml @@ -12,7 +12,7 @@ inputs: required: false default: "--only=code_quality --no-root" poetry_export_options: - description: Options for exporting dependencies to check for hash + description: Options for exporting dependencies to check for hash changes for cache invalidation required: false default: "--only=code_quality" @@ -33,4 +33,4 @@ runs: - name: Check code quality shell: bash run: poetry run ruff check . -... \ No newline at end of file +... diff --git a/actions/python/code-quality/spell-check/action.yaml b/actions/python/code-quality/spell-check/action.yaml index 8df3cba..019c03c 100644 --- a/actions/python/code-quality/spell-check/action.yaml +++ b/actions/python/code-quality/spell-check/action.yaml @@ -12,7 +12,7 @@ inputs: required: false default: "--only=code_quality --no-root" poetry_export_options: - description: Options for exporting dependencies to check for hash + description: Options for exporting dependencies to check for hash changes for cache invalidation required: false default: "--only=code_quality" @@ -33,4 +33,4 @@ runs: - name: Check spellings shell: bash run: poetry run typos . -... \ No newline at end of file +... diff --git a/actions/python/code-quality/type-check/action.yaml b/actions/python/code-quality/type-check/action.yaml index ee11bac..c505510 100644 --- a/actions/python/code-quality/type-check/action.yaml +++ b/actions/python/code-quality/type-check/action.yaml @@ -1,6 +1,6 @@ --- name: Type Check -description: Runs a type check on the codebase to ensure type safety and +description: Runs a type check on the codebase to ensure type safety and correctness. inputs: @@ -13,7 +13,7 @@ inputs: required: false default: "--with=code_quality --with=types --no-root" poetry_export_options: - description: Options for exporting dependencies to check for hash + description: Options for exporting dependencies to check for hash changes for cache invalidation required: false default: "--with=code_quality --with=types" @@ -39,4 +39,4 @@ runs: - name: Check types shell: bash run: poetry run mypy ${{ inputs.file_path }} -... \ No newline at end of file +... diff --git a/actions/python/code-test/integration-test/action.yaml b/actions/python/code-test/integration-test/action.yaml new file mode 100644 index 0000000..3a26f8e --- /dev/null +++ b/actions/python/code-test/integration-test/action.yaml @@ -0,0 +1,52 @@ +--- +name: Integration Test +description: Set up environment, run integration tests, and upload coverage report. + +inputs: + os: + default: ubuntu-latest + description: The operating system to use + python_version: + default: '3.11' + description: The version of Python to use + poetry_install_options: + default: '--with=test' + description: Options for installing dependencies via poetry + poetry_export_options: + default: '--with=test' + description: Options for exporting dependencies for cache invalidation + codecov_token: + description: Codecov token for uploading coverage reports + required: true + +runs: + using: composite + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up environment + uses: elixir-cloud-aai/actions/python/setup/poetry@main + with: + os: ${{ inputs.os }} + python_version: ${{ inputs.python_version }} + poetry_install_options: ${{ inputs.poetry_install_options }} + poetry_export_options: ${{ inputs.poetry_export_options }} + + - name: Run integration tests and generate coverage report + shell: bash + run: | + poetry run pytest \ + --cov-report term \ + --cov-report xml:test_integration.xml \ + --cov=tests/test_integration + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ inputs.codecov_token }} + flags: test_integration + files: ./test_integration.xml + fail_ci_if_error: true + verbose: true +... \ No newline at end of file diff --git a/actions/python/code-test/unit-test/action.yaml b/actions/python/code-test/unit-test/action.yaml new file mode 100644 index 0000000..2c53373 --- /dev/null +++ b/actions/python/code-test/unit-test/action.yaml @@ -0,0 +1,52 @@ +--- +name: Unit Test +description: Set up environment, run unit tests, and upload coverage report. + +inputs: + os: + default: ubuntu-latest + description: The operating system to use + python_version: + default: '3.11' + description: The version of Python to use + poetry_install_options: + default: '--with=test' + description: Options for installing dependencies via poetry + poetry_export_options: + default: '--with=test' + description: Options for exporting dependencies for cache invalidation + codecov_token: + description: Codecov token for uploading coverage reports + required: true + +runs: + using: composite + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up environment + uses: elixir-cloud-aai/actions/python/setup/poetry@main + with: + os: ${{ inputs.os }} + python_version: ${{ inputs.python_version }} + poetry_install_options: ${{ inputs.poetry_install_options }} + poetry_export_options: ${{ inputs.poetry_export_options }} + + - name: Run unit tests and generate coverage report + shell: bash + run: | + poetry run pytest \ + --cov-report term \ + --cov-report xml:test_unit.xml \ + --cov=tests/test_unit + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ inputs.codecov_token }} + flags: test_unit + files: ./test_unit.xml + fail_ci_if_error: true + verbose: true +... \ No newline at end of file diff --git a/actions/python/setup/poetry/action.yaml b/actions/python/setup/poetry/action.yaml index 42b80a6..556c926 100644 --- a/actions/python/setup/poetry/action.yaml +++ b/actions/python/setup/poetry/action.yaml @@ -17,7 +17,7 @@ inputs: description: Additional options to pass to poetry install poetry_export_options: default: '' - description: Options to pass to poetry export for hash generation for cache + description: Options to pass to poetry export for hash generation for cache invalidation runs: @@ -89,4 +89,4 @@ runs: if: steps.poetry-cache.outputs.cache-hit != 'true' shell: bash run: poetry install ${{ inputs.poetry_install_options }} --no-interaction -... \ No newline at end of file +... From 3c7bd44aa37cc7c829c87d2cec07e56c43c35fff Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Sun, 1 Sep 2024 17:23:58 +0530 Subject: [PATCH 4/9] vul --- .yamllint | 15 ++++++ .../code-test/integration-test/action.yaml | 18 +++++-- .../python/code-test/unit-test/action.yaml | 18 +++++-- actions/python/setup/poetry/action.yaml | 12 ++--- .../code-vulnerability/action.yaml | 50 +++++++++++++++++++ .../dependency-vulnerability/action.yaml | 40 +++++++++++++++ 6 files changed, 137 insertions(+), 16 deletions(-) create mode 100644 .yamllint create mode 100644 actions/python/vulnerability/code-vulnerability/action.yaml create mode 100644 actions/python/vulnerability/dependency-vulnerability/action.yaml diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..8320e79 --- /dev/null +++ b/.yamllint @@ -0,0 +1,15 @@ +--- +extends: default + +rules: + # Rule to enforce the use of double quotes + quoted-strings: + quote-type: double + required: false + # Additional rules can be configured as needed + line-length: + max: 120 + indentation: + spaces: 2 + document-start: disable +... \ No newline at end of file diff --git a/actions/python/code-test/integration-test/action.yaml b/actions/python/code-test/integration-test/action.yaml index 3a26f8e..2909de2 100644 --- a/actions/python/code-test/integration-test/action.yaml +++ b/actions/python/code-test/integration-test/action.yaml @@ -6,18 +6,26 @@ inputs: os: default: ubuntu-latest description: The operating system to use + required: false python_version: - default: '3.11' + default: "3.12" description: The version of Python to use + required: false poetry_install_options: - default: '--with=test' + default: "--with=test" description: Options for installing dependencies via poetry + required: false poetry_export_options: - default: '--with=test' + default: "--with=test" description: Options for exporting dependencies for cache invalidation + required: false codecov_token: description: Codecov token for uploading coverage reports required: true + file_path: + description: The path to the file or directory to integration test + required: false + default: "tests/test_integration" runs: using: composite @@ -39,7 +47,7 @@ runs: poetry run pytest \ --cov-report term \ --cov-report xml:test_integration.xml \ - --cov=tests/test_integration + --cov=${{ inputs.file_path }} - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 @@ -49,4 +57,4 @@ runs: files: ./test_integration.xml fail_ci_if_error: true verbose: true -... \ No newline at end of file +... diff --git a/actions/python/code-test/unit-test/action.yaml b/actions/python/code-test/unit-test/action.yaml index 2c53373..ad1827c 100644 --- a/actions/python/code-test/unit-test/action.yaml +++ b/actions/python/code-test/unit-test/action.yaml @@ -6,18 +6,26 @@ inputs: os: default: ubuntu-latest description: The operating system to use + required: false python_version: - default: '3.11' + default: "3.12" description: The version of Python to use + required: false poetry_install_options: - default: '--with=test' + default: "--with=test" description: Options for installing dependencies via poetry + required: false poetry_export_options: - default: '--with=test' + default: "--with=test" description: Options for exporting dependencies for cache invalidation + required: false codecov_token: description: Codecov token for uploading coverage reports required: true + file_path: + description: The path to the file or directory to unit test + required: false + default: "tests/test_unit" runs: using: composite @@ -39,7 +47,7 @@ runs: poetry run pytest \ --cov-report term \ --cov-report xml:test_unit.xml \ - --cov=tests/test_unit + --cov=${{ inputs.file_path }} - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 @@ -49,4 +57,4 @@ runs: files: ./test_unit.xml fail_ci_if_error: true verbose: true -... \ No newline at end of file +... diff --git a/actions/python/setup/poetry/action.yaml b/actions/python/setup/poetry/action.yaml index 556c926..c8243f6 100644 --- a/actions/python/setup/poetry/action.yaml +++ b/actions/python/setup/poetry/action.yaml @@ -7,26 +7,26 @@ inputs: default: ubuntu-latest description: The operating system to use python_version: - default: '3.11' + default: "3.12" description: The version of Python to use poetry_version: - default: '1.8.2' + default: "1.8.2" description: The version of Poetry to install poetry_install_options: - default: '' + default: "" description: Additional options to pass to poetry install poetry_export_options: - default: '' + default: "" description: Options to pass to poetry export for hash generation for cache invalidation runs: using: composite steps: - - uses: 'actions/setup-python@v5' + - uses: "actions/setup-python@v5" id: setup-python with: - python-version: '${{ inputs.python_version }}' + python-version: "${{ inputs.python_version }}" - name: Setup pipx environment Variables id: pipx-env-setup diff --git a/actions/python/vulnerability/code-vulnerability/action.yaml b/actions/python/vulnerability/code-vulnerability/action.yaml new file mode 100644 index 0000000..428b849 --- /dev/null +++ b/actions/python/vulnerability/code-vulnerability/action.yaml @@ -0,0 +1,50 @@ +--- +name: Code Vulnerability Test +description: Set up environment, run code vulnerability checks, and report + findings. + +inputs: + os: + default: ubuntu-latest + description: The operating system to use + required: false + python_version: + default: "3.12" + description: The version of Python to use + required: false + poetry_install_options: + default: "--only=vulnerability --no-root" + description: Options for installing dependencies via poetry + required: false + poetry_export_options: + default: "--only=vulnerability" + description: Options for exporting dependencies for cache invalidation + required: false + config_path: + description: The path to the Bandit configuration file + default: "pyproject.toml" + required: false + file_path: + description: The path to the file or directory to check for vulnerabilities + required: true + +runs: + using: composite + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up environment + uses: elixir-cloud-aai/actions/python/setup/poetry@main + with: + os: ${{ inputs.os }} + python_version: ${{ inputs.python_version }} + poetry_install_options: ${{ inputs.poetry_install_options }} + poetry_export_options: ${{ inputs.poetry_export_options }} + + - name: Check code vulnerabilities with Bandit + shell: bash + run: poetry run bandit \ + -c ${{ inputs.config_path }} \ + -r ${{ inputs.file_path }} +... \ No newline at end of file diff --git a/actions/python/vulnerability/dependency-vulnerability/action.yaml b/actions/python/vulnerability/dependency-vulnerability/action.yaml new file mode 100644 index 0000000..e9a4ed5 --- /dev/null +++ b/actions/python/vulnerability/dependency-vulnerability/action.yaml @@ -0,0 +1,40 @@ +--- +name: Dependency Vulnerability Test +description: Set up environment, run dependency vulnerability checks, and report findings. + +inputs: + os: + default: ubuntu-latest + description: The operating system to use + required: false + python_version: + default: "3.12" + description: The version of Python to use + required: false + poetry_install_options: + default: "--only=vulnerability --no-root" + description: Options for installing dependencies via poetry + required: false + poetry_export_options: + default: "--only=vulnerability" + description: Options for exporting dependencies for cache invalidation + required: false + +runs: + using: composite + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up environment + uses: elixir-cloud-aai/actions/python/setup/poetry@main + with: + os: ${{ inputs.os }} + python_version: ${{ inputs.python_version }} + poetry_install_options: ${{ inputs.poetry_install_options }} + poetry_export_options: ${{ inputs.poetry_export_options }} + + - name: Check dependency vulnerabilities with Safety + shell: bash + run: poetry run safety check --full-report +... \ No newline at end of file From 7d699d8e516f15e3793a51cd261cfc326d7b1bfe Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Sun, 1 Sep 2024 17:45:17 +0530 Subject: [PATCH 5/9] docs check so that each pr has uptodate docs --- .yamllint | 2 +- .../docs/autogenerated-docs-check/action.yaml | 64 +++++++++++++++++++ actions/python/release/pypi/action.yaml | 42 ++++++++++++ .../code-vulnerability/action.yaml | 4 +- .../dependency-vulnerability/action.yaml | 2 +- 5 files changed, 110 insertions(+), 4 deletions(-) create mode 100644 actions/python/docs/autogenerated-docs-check/action.yaml create mode 100644 actions/python/release/pypi/action.yaml diff --git a/.yamllint b/.yamllint index 8320e79..b082d5a 100644 --- a/.yamllint +++ b/.yamllint @@ -12,4 +12,4 @@ rules: indentation: spaces: 2 document-start: disable -... \ No newline at end of file +... diff --git a/actions/python/docs/autogenerated-docs-check/action.yaml b/actions/python/docs/autogenerated-docs-check/action.yaml new file mode 100644 index 0000000..0bfbceb --- /dev/null +++ b/actions/python/docs/autogenerated-docs-check/action.yaml @@ -0,0 +1,64 @@ +--- +name: Documentation Check +description: Verifies that the API documentation is up to date. + +inputs: + os: + default: ubuntu-latest + description: The operating system to use + required: false + python_version: + default: "3.12" + description: The version of Python to use + required: false + poetry_install_options: + default: "--with=docs --with=types --no-root" + description: Options to pass to poetry install + required: false + poetry_export_options: + default: "--with=docs --with=types" + description: Options to pass to poetry export for cache invalidation + required: false + file_path: + description: File path for which the docs will be generated + required: true + auto_doc_dir: + description: Directory path that stores all the auto generated docs for + the packages + default: "/docs/source/pages" + required: false + +runs: + using: composite + steps: + - uses: actions/checkout@v4 + + - name: Set up environment + uses: elixir-cloud-aai/actions/python/setup/poetry@main + with: + os: ${{ inputs.os }} + python-version: ${{ inputs.python_version }} + poetry-install-options: ${{ inputs.poetry_install_options }} + poetry-export-options: ${{ inputs.poetry_export_options }} + + - name: Generate API docs + shell: bash + run: | + poetry run sphinx-apidoc \ + -o /tmp/docs/source/pages \ + ${{ inputs.file_path }}/ + + - name: Compare current docs with latest docs + shell: bash + run: | + shasum /tmp/docs/source/pages/* > /tmp/docs.sha + shasum ${{ inputs.auto_doc_dir }}/* > docs/project_doc.sha + awk '{print $1}' /tmp/docs.sha > /tmp/docs_hashes.sha + awk '{print $1}' docs/project_doc.sha > docs/project_doc_hashes.sha + diff=$(diff /tmp/docs_hashes.sha docs/project_doc_hashes.sha) || true + if [[ -n "$diff" ]]; then + echo "::error::API documentation is out of date." + exit 1 + else + echo "API documentation is up to date." +... diff --git a/actions/python/release/pypi/action.yaml b/actions/python/release/pypi/action.yaml new file mode 100644 index 0000000..25fecf9 --- /dev/null +++ b/actions/python/release/pypi/action.yaml @@ -0,0 +1,42 @@ +--- +name: Publish to PyPI +description: Publishes the Python package to PyPI when a release is made. + +inputs: + os: + default: ubuntu-latest + description: The operating system to use + required: false + python_version: + default: "3.12" + description: The version of Python to use + required: false + poetry_install_options: + default: "--only=main" + description: Additional options to pass to poetry install + required: false + poetry_export_options: + default: "--only=main" + description: Options to pass to poetry export for cache invalidation + required: false + pypi_token: + description: PYPI token for release the package + required: true + +runs: + using: composite + steps: + - uses: actions/checkout@v4 + + - name: Set up environment + uses: elixir-cloud-aai/actions/python/setup/poetry@main + with: + os: ${{ inputs.os }} + python-version: ${{ inputs.python_version }} + poetry-install-options: ${{ inputs.poetry_install_options }} + poetry-export-options: ${{ inputs.poetry_export_options }} + + - name: Publish package to PyPI + shell: bash + run: poetry publish --build -u __token__ -p ${{ inputs.pypi_token }} +... diff --git a/actions/python/vulnerability/code-vulnerability/action.yaml b/actions/python/vulnerability/code-vulnerability/action.yaml index 428b849..b3022f5 100644 --- a/actions/python/vulnerability/code-vulnerability/action.yaml +++ b/actions/python/vulnerability/code-vulnerability/action.yaml @@ -1,6 +1,6 @@ --- name: Code Vulnerability Test -description: Set up environment, run code vulnerability checks, and report +description: Set up environment, run code vulnerability checks, and report findings. inputs: @@ -47,4 +47,4 @@ runs: run: poetry run bandit \ -c ${{ inputs.config_path }} \ -r ${{ inputs.file_path }} -... \ No newline at end of file +... diff --git a/actions/python/vulnerability/dependency-vulnerability/action.yaml b/actions/python/vulnerability/dependency-vulnerability/action.yaml index e9a4ed5..a21026b 100644 --- a/actions/python/vulnerability/dependency-vulnerability/action.yaml +++ b/actions/python/vulnerability/dependency-vulnerability/action.yaml @@ -37,4 +37,4 @@ runs: - name: Check dependency vulnerabilities with Safety shell: bash run: poetry run safety check --full-report -... \ No newline at end of file +... From f38e1813000eb92169bf83af67eaaa99a038f6cc Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Sun, 1 Sep 2024 17:46:39 +0530 Subject: [PATCH 6/9] format --- actions/python/code-quality/type-check/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/python/code-quality/type-check/action.yaml b/actions/python/code-quality/type-check/action.yaml index c505510..34adb9e 100644 --- a/actions/python/code-quality/type-check/action.yaml +++ b/actions/python/code-quality/type-check/action.yaml @@ -20,7 +20,7 @@ inputs: file_path: description: The path to the file or directory to type check required: false - default: " . " + default: "." runs: From b2b2861f6c86b8236b62bc9e15f38b73ae2faf76 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Mon, 16 Sep 2024 14:25:15 +0530 Subject: [PATCH 7/9] remove spell checker --- .../code-quality/spell-check/action.yaml | 36 ------------------- actions/python/release/pypi/action.yaml | 6 +++- 2 files changed, 5 insertions(+), 37 deletions(-) delete mode 100644 actions/python/code-quality/spell-check/action.yaml diff --git a/actions/python/code-quality/spell-check/action.yaml b/actions/python/code-quality/spell-check/action.yaml deleted file mode 100644 index 019c03c..0000000 --- a/actions/python/code-quality/spell-check/action.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -name: Spell Check -description: Runs a spell check on the codebase to identify spelling errors. - -inputs: - python_version: - description: Python version to use - default: "3.12" - required: false - poetry_install_options: - description: Options for installing dependencies via poetry - required: false - default: "--only=code_quality --no-root" - poetry_export_options: - description: Options for exporting dependencies to check for hash - changes for cache invalidation - required: false - default: "--only=code_quality" - -runs: - using: composite - steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Set up environment - uses: elixir-cloud-aai/actions/python/setup/poetry@main - with: - python_version: ${{ inputs.python_version }} - poetry_install_options: ${{ inputs.poetry_install_options }} - poetry_export_options: ${{ inputs.poetry_export_options }} - - - name: Check spellings - shell: bash - run: poetry run typos . -... diff --git a/actions/python/release/pypi/action.yaml b/actions/python/release/pypi/action.yaml index 25fecf9..617a874 100644 --- a/actions/python/release/pypi/action.yaml +++ b/actions/python/release/pypi/action.yaml @@ -36,7 +36,11 @@ runs: poetry-install-options: ${{ inputs.poetry_install_options }} poetry-export-options: ${{ inputs.poetry_export_options }} + - name: Build package + shell: bash + run: poetry build + - name: Publish package to PyPI shell: bash - run: poetry publish --build -u __token__ -p ${{ inputs.pypi_token }} + run: poetry publish -u __token__ -p ${{ inputs.pypi_token }} ... From 9b45748cdaf0237a373111f0a4375f92d390bbd8 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Wed, 2 Oct 2024 02:29:46 +0530 Subject: [PATCH 8/9] feat: add commenting functionality --- .../python/code-quality/format/action.yaml | 53 ++++++++++++ actions/python/code-quality/lint/action.yaml | 50 +++++++++++ .../code-quality/type-check/action.yaml | 73 +++++++++++++++++ .../code-test/integration-test/action.yaml | 56 ++++++++++++- .../python/code-test/unit-test/action.yaml | 54 +++++++++++- actions/python/comment/action.yaml | 76 +++++++++++++++++ .../docs/autogenerated-docs-check/action.yaml | 82 ++++++++++++++++++- .../code-vulnerability/action.yaml | 52 ++++++++++++ .../dependency-vulnerability/action.yaml | 52 ++++++++++++ 9 files changed, 542 insertions(+), 6 deletions(-) create mode 100644 actions/python/comment/action.yaml diff --git a/actions/python/code-quality/format/action.yaml b/actions/python/code-quality/format/action.yaml index 8ebc963..8f841ab 100644 --- a/actions/python/code-quality/format/action.yaml +++ b/actions/python/code-quality/format/action.yaml @@ -4,6 +4,10 @@ description: Checks and ensures the code is formatted according to the defined style guidelines. inputs: + os: + description: The operating system to use + default: ubuntu-latest + required: false python_version: description: Python version to use default: "3.12" @@ -26,12 +30,61 @@ runs: - name: Set up environment uses: elixir-cloud-aai/actions/python/setup/poetry@main + id: setup + continue-on-error: true with: + os: ${{ inputs.os }} python_version: ${{ inputs.python_version }} poetry_install_options: ${{ inputs.poetry_install_options }} poetry_export_options: ${{ inputs.poetry_export_options }} + - name: Comment on PR + if: steps.setup.outcome == 'failure' + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: format + step_name: setup + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + Check your `pyproject.toml` and the dependency groups in them, + especially the `code_quality` group. + +
+ Env used for CI + OS: ${{ inputs.os }} + Python Version: ${{ inputs.python_version }} + Poetry Install Options: ${{ inputs.poetry_install_options }} + Poetry Export Options: ${{ inputs.poetry_export_options }} +
+ - name: Check code style + id: format + continue-on-error: true shell: bash run: poetry run ruff format --check + + - name: Comment on PR + if: steps.format.outcome == 'failure' + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: format + step_name: format + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + With the environment set up and dependencies installed, + run `poetry run ruff format` to check and fix some code style issues. + +
+ Use Makefile command + Default makefile from cookiecutter template has a command + `make fl` to run the check. + + Run make in the root directory of the project, to see all the + available commands. +
... +q \ No newline at end of file diff --git a/actions/python/code-quality/lint/action.yaml b/actions/python/code-quality/lint/action.yaml index b585cdb..e5132b9 100644 --- a/actions/python/code-quality/lint/action.yaml +++ b/actions/python/code-quality/lint/action.yaml @@ -3,6 +3,10 @@ name: Lint Code description: Lints the code to ensure code quality and adherence to standards. inputs: + os: + description: The operating system to use + default: ubuntu-latest + required: false python_version: description: Python version to use default: "3.12" @@ -25,12 +29,58 @@ runs: - name: Set up environment uses: elixir-cloud-aai/actions/python/setup/poetry@main + continue-on-error: true with: + os: ${{ inputs.os }} python_version: ${{ inputs.python_version }} poetry_install_options: ${{ inputs.poetry_install_options }} poetry_export_options: ${{ inputs.poetry_export_options }} + - name: Comment on PR + if: steps.setup.outcome == 'failure' + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: lint + step_name: setup + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + Check your `pyproject.toml` and the dependency groups in them. + +
+ Env used for CI + OS: ${{ inputs.os }} + Python Version: ${{ inputs.python_version }} + Poetry Install Options: ${{ inputs.poetry_install_options }} + Poetry Export Options: ${{ inputs.poetry_export_options }} +
+ - name: Check code quality shell: bash + id: lint run: poetry run ruff check . + + - name: Comment on PR + if: steps.lint.outcome == 'failure' + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: format + step_name: format + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + With the environment set up and dependencies installed, + run `poetry run ruff check --fix` to check and fix some lint + issues. + +
+ Use Makefile command + Default makefile from cookiecutter template has a command + `make fl` to run the check. + + Run make in the root directory of the project, to see all the + available commands. +
... diff --git a/actions/python/code-quality/type-check/action.yaml b/actions/python/code-quality/type-check/action.yaml index 34adb9e..d17930a 100644 --- a/actions/python/code-quality/type-check/action.yaml +++ b/actions/python/code-quality/type-check/action.yaml @@ -4,6 +4,10 @@ description: Runs a type check on the codebase to ensure type safety and correctness. inputs: + os: + description: The operating system to use + required: false + default: "ubuntu-latest" python_version: description: Python version to use default: "3.12" @@ -31,12 +35,81 @@ runs: - name: Set up environment uses: elixir-cloud-aai/actions/python/setup/poetry@main + id: setup + continue-on-error: true with: + os: ${{ inputs.os }} python_version: ${{ inputs.python_version }} poetry_install_options: ${{ inputs.poetry_install_options }} poetry_export_options: ${{ inputs.poetry_export_options }} + - name: Comment on PR + if: steps.setup.outcome == 'failure' + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: type-check + step_name: setup + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + Check your `pyproject.toml` and the dependency groups in them, + especially `code_quality` and `types` group. + +
+ Env used for CI + OS: ${{ inputs.os }} + Python Version: ${{ inputs.python_version }} + Poetry Install Options: ${{ inputs.poetry_install_options }} + Poetry Export Options: ${{ inputs.poetry_export_options }} +
+ - name: Check types + id: type-check + continue-on-error: true shell: bash run: poetry run mypy ${{ inputs.file_path }} + + - name: Comment on PR + if: steps.type-check.outcome == 'failure' + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: type-check + step_name: type-check + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + With the environment set up and dependencies installed, + run `run mypy ${{ inputs.file_path }}` to see mypy errors. + +
+ Missing stubs or types + If you see errors related to missing stubs or types, + try running `poetry add types- --group=types` or + `poetry add -stub --group=types` to add type stubs. + + If that fails, either the package does not have types/stubs + or they have different name. + + In that case you can seach for them and add them to types group. +
+ +
+ Ignore error + Add the module name in pyproject.toml under [tools.mypy.overrides] + section inside modules list. + + ```toml + [[tool.mypy.overrides]] + ignore_missing_imports = true + module = [ + "connexion.*", + ] + ``` + + Try to as granular as possible to avoid ignoring all errors. If + you are using `x.y.z` import, add `x.y.z.*` to the list instead + of `x.*`. +
... diff --git a/actions/python/code-test/integration-test/action.yaml b/actions/python/code-test/integration-test/action.yaml index 2909de2..aecf79f 100644 --- a/actions/python/code-test/integration-test/action.yaml +++ b/actions/python/code-test/integration-test/action.yaml @@ -35,13 +35,38 @@ runs: - name: Set up environment uses: elixir-cloud-aai/actions/python/setup/poetry@main + continue-on-error: true + id: setup with: os: ${{ inputs.os }} python_version: ${{ inputs.python_version }} poetry_install_options: ${{ inputs.poetry_install_options }} poetry_export_options: ${{ inputs.poetry_export_options }} + - name: Comment on PR + if: steps.setup.outcome == 'failure' + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: integration-test + step_name: setup + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + Test depend on the main package dependecies and `test` + group in `pyproject.toml`. + +
+ Env used for CI + OS: ${{ inputs.os }} + Python Version: ${{ inputs.python_version }} + Poetry Install Options: ${{ inputs.poetry_install_options }} + Poetry Export Options: ${{ inputs.poetry_export_options }} +
+ - name: Run integration tests and generate coverage report + id: test + continue-on-error: true shell: bash run: | poetry run pytest \ @@ -49,12 +74,41 @@ runs: --cov-report xml:test_integration.xml \ --cov=${{ inputs.file_path }} + - name: Comment on PR + if: steps.upload.outcome == 'failure' + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: integration-test + step_name: test + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + Check your integration tests in ${{ inputs.file_path }}, + and the dependencies in `pyproject.toml`. + + Run `poetry run pytest --cov=${{ inputs.file_path }}` to see the errors. + - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 + id: upload with: token: ${{ inputs.codecov_token }} flags: test_integration files: ./test_integration.xml - fail_ci_if_error: true + fail_ci_if_error: false verbose: true + + - name: Comment on PR + if: ${{ inputs.codecov_token }} == null || ${{ inputs.codecov_token }} == '' + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: integration-test + step_name: upload + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + Codecov token is missing. Please ask admin to add the token to the + secrets. The coverage report was generated but not uploaded. ... diff --git a/actions/python/code-test/unit-test/action.yaml b/actions/python/code-test/unit-test/action.yaml index ad1827c..1bca36c 100644 --- a/actions/python/code-test/unit-test/action.yaml +++ b/actions/python/code-test/unit-test/action.yaml @@ -34,6 +34,7 @@ runs: uses: actions/checkout@v4 - name: Set up environment + id: setup uses: elixir-cloud-aai/actions/python/setup/poetry@main with: os: ${{ inputs.os }} @@ -41,7 +42,29 @@ runs: poetry_install_options: ${{ inputs.poetry_install_options }} poetry_export_options: ${{ inputs.poetry_export_options }} + - name: Comment on PR + if: steps.setup.outcome == 'failure' + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: unit-test + step_name: setup + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + Test depend on the main package dependecies and `test` + group in `pyproject.toml`. + +
+ Env used for CI + OS: ${{ inputs.os }} + Python Version: ${{ inputs.python_version }} + Poetry Install Options: ${{ inputs.poetry_install_options }} + Poetry Export Options: ${{ inputs.poetry_export_options }} +
+ - name: Run unit tests and generate coverage report + id: unit-test shell: bash run: | poetry run pytest \ @@ -49,12 +72,41 @@ runs: --cov-report xml:test_unit.xml \ --cov=${{ inputs.file_path }} + - name: Comment on PR + if: steps.unit-test.outcome == 'failure' + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: unit-test + step_name: test + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + Check your unit tests in ${{ inputs.file_path }}, + and the dependencies in `pyproject.toml`. + + Run `poetry run pytest --cov=${{ inputs.file_path }}` to see the errors. + - name: Upload coverage to Codecov + id: upload uses: codecov/codecov-action@v4 with: token: ${{ inputs.codecov_token }} flags: test_unit files: ./test_unit.xml - fail_ci_if_error: true + fail_ci_if_error: false verbose: true + + - name: Comment on PR + if: ${{ inputs.codecov_token }} == null || ${{ inputs.codecov_token }} == '' + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: unit-test + step_name: upload + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + Codecov token is missing. Please ask admin to add the token to the + secrets. The coverage report was generated but not uploaded. ... diff --git a/actions/python/comment/action.yaml b/actions/python/comment/action.yaml new file mode 100644 index 0000000..3edf603 --- /dev/null +++ b/actions/python/comment/action.yaml @@ -0,0 +1,76 @@ +--- +name: Commenting bot to report CI failures +description: | + This action is used to comment on a pull request with the CI failure details. + It checks if the comment is already present and updates the comment with the + latest CI failure details. + +inputs: + ci_name: + description: The name of the CI + required: true + commit_id: + description: The commit id to use + required: true + error-after-comment: + description: Stop the workflow after commenting + required: false + default: "true" + issue-number: + description: The issue number to comment on + required: true + message: + description: The message to include in the comment + required: true + step_name: + description: The name of the step the failed in the + required: true + username: + description: The username to tag in the message + default: jaeaeich + required: false + +runs: + using: composite + steps: + - name: Create the title of the comment + id: globals + shell: bash + run: echo "TITLE=Hey @$INPUT_USERNAME, \`$INPUT_CI_NAME\` CI failed at \`$INPUT_STEP_NAME\` step!" >> "${GITHUB_OUTPUT}" + env: + INPUT_USERNAME: ${{ inputs.username }} + INPUT_CI_NAME: ${{ inputs.ci_name }} + INPUT_STEP_NAME: ${{ inputs.step_name }} + + - name: Find if comment is already present + uses: peter-evans/find-comment@v3 + id: find-comment + with: + issue-number: ${{ inputs.issue-number }} + comment-author: 'github-actions[bot]' + body-includes: ${{ steps.globals.outputs.TITLE }} + + - name: Create a new comment if there hasn't been one + if: steps.find-comment.outputs.comment-id == '' + id: fresh-comment + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ inputs.issue-number }} + body: | + ${{ steps.globals.outputs.TITLE }} + ${{ inputs.message }} + + - name: Update the comment, to remind user + if: steps.find-comment.outputs.comment-id != '' + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.find-comment.outputs.comment-id }} + issue-number: ${{ inputs.issue-number }} + body: | + Hey @${{ inputs.username }}, it failed again for commit ${{ inputs.commit_id }}! + + - name: Fail the workflow if error-after-comment is true + if: ${{ inputs.error-after-comment }} + shell: bash + run: exit 1 +... \ No newline at end of file diff --git a/actions/python/docs/autogenerated-docs-check/action.yaml b/actions/python/docs/autogenerated-docs-check/action.yaml index 0bfbceb..d13483c 100644 --- a/actions/python/docs/autogenerated-docs-check/action.yaml +++ b/actions/python/docs/autogenerated-docs-check/action.yaml @@ -20,8 +20,9 @@ inputs: description: Options to pass to poetry export for cache invalidation required: false file_path: - description: File path for which the docs will be generated - required: true + description: File path for which the docs will be generated, default to the + package name from the pyproject.toml + required: false auto_doc_dir: description: Directory path that stores all the auto generated docs for the packages @@ -31,25 +32,76 @@ inputs: runs: using: composite steps: - - uses: actions/checkout@v4 + - name: Check out repository + uses: actions/checkout@v4 - name: Set up environment + id: setup uses: elixir-cloud-aai/actions/python/setup/poetry@main + continue-on-error: true with: os: ${{ inputs.os }} python-version: ${{ inputs.python_version }} poetry-install-options: ${{ inputs.poetry_install_options }} poetry-export-options: ${{ inputs.poetry_export_options }} + - name: Comment on PR + if: steps.setup.outcome == 'failure' + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: docs + step_name: setup + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + Check your `pyproject.toml` and the dependency groups in them, + especially `docs` and `types` group. + +
+ Env used for CI + OS: ${{ inputs.os }} + Python Version: ${{ inputs.python_version }} + Poetry Install Options: ${{ inputs.poetry_install_options }} + Poetry Export Options: ${{ inputs.poetry_export_options }} +
+ + - name: Get the file_path if user doesn't provide it + id: get-file-path + shell: bash + run: | + if [[ -z "${{ inputs.file_path }}" ]]; then + file_path=$(poetry version | awk '{print $1}') + else + file_path="${{ inputs.file_path }}" + fi + echo "file_path=$file_path" >> $GITHUB_ENV + + # This should not be possible, but just in case + - name: Comment on PR + if: ${{ env.file_path }} == "" + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: docs + step_name: get-file-path + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + Couldn't retrieve project name from `pyproject.toml` and + `file_path`` is not provided for the workflow. + - name: Generate API docs shell: bash run: | poetry run sphinx-apidoc \ -o /tmp/docs/source/pages \ - ${{ inputs.file_path }}/ + ${{ env.file_path }}/ - name: Compare current docs with latest docs shell: bash + id: compare-docs + continue-on-error: true run: | shasum /tmp/docs/source/pages/* > /tmp/docs.sha shasum ${{ inputs.auto_doc_dir }}/* > docs/project_doc.sha @@ -61,4 +113,26 @@ runs: exit 1 else echo "API documentation is up to date." + + - name: Comment on PR + if: steps.compare-docs.outcome == 'failure' + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: docs + step_name: compare-docs + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + API documentation is out of date. Run + `poetry run sphinx-apidoc ${{ env.file_path }}` to update the docs. + + Note: You can also delete the auto generated docs and regenerate + them if there has been a major change in the codebase. + +
+ Use Makefile command + Default makefile from cookiecutter template has a command + `make docs` to run the sphinx-apidoc. +
... diff --git a/actions/python/vulnerability/code-vulnerability/action.yaml b/actions/python/vulnerability/code-vulnerability/action.yaml index b3022f5..37882a3 100644 --- a/actions/python/vulnerability/code-vulnerability/action.yaml +++ b/actions/python/vulnerability/code-vulnerability/action.yaml @@ -36,15 +36,67 @@ runs: - name: Set up environment uses: elixir-cloud-aai/actions/python/setup/poetry@main + id: setup + continue-on-error: false with: os: ${{ inputs.os }} python_version: ${{ inputs.python_version }} poetry_install_options: ${{ inputs.poetry_install_options }} poetry_export_options: ${{ inputs.poetry_export_options }} + - name: Comment on PR + if: steps.setup.outcome == 'failure' + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: bandit + step_name: setup + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + Make sure bandit is in `vulnerability` group in `pyproject.toml`. + +
+ Env used for CI + OS: ${{ inputs.os }} + Python Version: ${{ inputs.python_version }} + Poetry Install Options: ${{ inputs.poetry_install_options }} + Poetry Export Options: ${{ inputs.poetry_export_options }} +
+ - name: Check code vulnerabilities with Bandit + id: check + continue-on-error: true shell: bash run: poetry run bandit \ -c ${{ inputs.config_path }} \ -r ${{ inputs.file_path }} + + - name: Comment on PR + if: steps.check.outcome == 'failure' + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: bandit + step_name: check + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + Check your code for vulnerabilities using Bandit. + + Run `poetry run bandit -c ${{ inputs.config_path }} -r ${{ inputs.file_path }}` + to check for errors. + +
+ Ignore errors + If you are sure that the error is a false positive, you can add the + error code to the `skips` list in `pyproject.toml`. + + ```toml + [tool.bandit] + skips = [ + "B108" # Insecure usage of temp file/directory, false positive. + ] + ``` +
... diff --git a/actions/python/vulnerability/dependency-vulnerability/action.yaml b/actions/python/vulnerability/dependency-vulnerability/action.yaml index a21026b..119ab93 100644 --- a/actions/python/vulnerability/dependency-vulnerability/action.yaml +++ b/actions/python/vulnerability/dependency-vulnerability/action.yaml @@ -27,6 +27,8 @@ runs: uses: actions/checkout@v4 - name: Set up environment + id: setup + continue-on-error: true uses: elixir-cloud-aai/actions/python/setup/poetry@main with: os: ${{ inputs.os }} @@ -34,7 +36,57 @@ runs: poetry_install_options: ${{ inputs.poetry_install_options }} poetry_export_options: ${{ inputs.poetry_export_options }} + - name: Comment on PR + if: steps.setup.outcome == 'failure' + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: safety + step_name: setup + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + Make sure bandit is in `vulnerability` group in `pyproject.toml`. + +
+ Env used for CI + OS: ${{ inputs.os }} + Python Version: ${{ inputs.python_version }} + Poetry Install Options: ${{ inputs.poetry_install_options }} + Poetry Export Options: ${{ inputs.poetry_export_options }} +
+ - name: Check dependency vulnerabilities with Safety shell: bash + id: check + continue-on-error: true run: poetry run safety check --full-report + + - name: Comment on PR + if: steps.check.outcome == 'failure' + uses: elixir-cloud-aai/actions/github/comment@main + with: + ci_name: safety + step_name: check + commit_id: ${{ github.sha }} + issue-number: ${{ github.event.number }} + username: ${{ github.actor}} + message: | + Check your code for vulnerabilities using safety cli. + + Run `poetry run safety check --full-report` to check for errors. + +
+ Ignore errors + If you are sure that the error is a false positive, you can add the + error code to `.safety-policy.yaml`. + + ```yaml + security: + ignore-vulnerabilities: + ERROR_CODE: + reason: 'REASON' + expires: 'DATE' + ``` +
... From ff60c28be78f9450af843378c3b40f257831096d Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Sun, 6 Oct 2024 21:31:52 +0530 Subject: [PATCH 9/9] format --- actions/python/code-quality/format/action.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/actions/python/code-quality/format/action.yaml b/actions/python/code-quality/format/action.yaml index 8f841ab..824d25a 100644 --- a/actions/python/code-quality/format/action.yaml +++ b/actions/python/code-quality/format/action.yaml @@ -86,5 +86,4 @@ runs: Run make in the root directory of the project, to see all the available commands. -... -q \ No newline at end of file +... \ No newline at end of file