From 3907e2188d4644fbe994ce970182240b87ba10ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Thu, 25 Apr 2024 10:44:29 +0100 Subject: [PATCH] ci: separate workflows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dune cache inside 'setup-ocaml' is trimmed to 5GB/jobcount, which is currently about 10. But not all jobs here use the dune cache, so separate the workflows. (Also the dune cache will get compressed afterwards). The SDK build does use the dune cache, but eventually it might become smaller, e.g. if we change it to install just the dependencies needed for SDK gen, and not everything. It does have a lot of unrelated jobs though. The workflows need a unique name, otherwise they cancel each-other. The dune cache is ~1.4GB now compressed, which should speed up the setup-ocaml step on next run as more dependencies will be cached. Setup XenAPI environment now only takes 2m51s instead of 5m55s. Signed-off-by: Edwin Török --- .github/workflows/main.yml | 126 -------------------------------- .github/workflows/other.yml | 138 ++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 126 deletions(-) create mode 100644 .github/workflows/other.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ada260060ad..5f2dd84a112 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,109 +12,6 @@ concurrency: # On new push, cancel old workflows from the same PR, branch or tag cancel-in-progress: true jobs: - python-test: - name: Python tests - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - python-version: ["2.7", "3.11"] - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # To check which files changed: origin/master..HEAD - - uses: LizardByte/setup-python-action@master - with: - python-version: ${{matrix.python-version}} - - - uses: actions/cache@v4 - name: Setup cache for running pre-commit fast - with: - path: ~/.cache/pre-commit - key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} - - - run: echo "::add-matcher::.github/workflows/python-warning-matcher.json" - name: "Setup GitHub for reporting Python warnings as annotations in pull request code review" - - - uses: pre-commit/action@v3.0.1 - name: Run pre-commit checks (no spaces at end of lines, etc) - if: ${{ matrix.python-version != '2.7' }} - with: - extra_args: --all-files --verbose --hook-stage commit - env: - SKIP: no-commit-to-branch - - - name: Install dependencies only needed for python 2 - if: ${{ matrix.python-version == '2.7' }} - run: pip install enum - - - name: Install dependencies only needed for python 3 - if: ${{ matrix.python-version != '2.7' }} - run: pip install opentelemetry-api opentelemetry-exporter-zipkin-json opentelemetry-sdk pandas pytype toml wrapt - - - name: Install common dependencies for Python ${{matrix.python-version}} - run: pip install future mock pytest-coverage pytest-mock - - - name: Run Pytest for python 2 and get code coverage for Codecov - if: ${{ matrix.python-version == '2.7' }} - run: > - pytest - --cov=scripts --cov=ocaml/xcp-rrdd - scripts/ ocaml/xcp-rrdd -vv -rA - --junitxml=.git/pytest${{matrix.python-version}}.xml - --cov-report term-missing - --cov-report xml:.git/coverage${{matrix.python-version}}.xml - env: - PYTHONDEVMODE: yes - - - name: Run Pytest for python 3 and get code coverage for Codecov - if: ${{ matrix.python-version != '2.7' }} - run: > - pytest - --cov=scripts --cov=ocaml/xcp-rrdd --cov=python3/ - scripts/ ocaml/xcp-rrdd python3/ -vv -rA - --junitxml=.git/pytest${{matrix.python-version}}.xml - --cov-report term-missing - --cov-report xml:.git/coverage${{matrix.python-version}}.xml - env: - PYTHONDEVMODE: yes - - - name: Upload Python ${{matrix.python-version}} coverage report to Codecov - uses: codecov/codecov-action@v3 - with: - directory: .git - files: coverage${{matrix.python-version}}.xml - env_vars: OS,PYTHON - fail_ci_if_error: false - flags: python${{matrix.python-version}} - name: coverage${{matrix.python-version}} - verbose: true - - - uses: dciborow/action-pylint@0.1.0 - if: ${{ matrix.python-version != '2.7' }} - with: - reporter: github-pr-review - level: warning - # To be customized to cover remaining Python scripts: - glob_pattern: "**/*.py" - continue-on-error: true - - - name: Run pytype checks - if: ${{ matrix.python-version != '2.7' }} - run: ./pytype_reporter.py - env: - PR_NUMBER: ${{ github.event.number }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PYTYPE_REPORTER_DEBUG: True - - - name: pyflakes - uses: reviewdog/action-pyflakes@v1 - with: - github_token: ${{ secrets.github_token }} - continue-on-error: true - - ocaml-tests: name: Run OCaml tests runs-on: ubuntu-22.04 @@ -152,26 +49,3 @@ jobs: - name: Check disk space run: df -h || true - - deprecation-test: - name: Deprecation tests - runs-on: ubuntu-22.04 - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Generate empty configuration for make to be happy - run: touch config.mk - - - name: quality-gate - run: make quality-gate - - test-sdk-builds: - name: Test SDK builds - uses: ./.github/workflows/generate-and-build-sdks.yml - with: - # Ensure you also update ocaml-tests - # when changing this value, to keep builds - # consistent - xapi_version: "v0.0.0" diff --git a/.github/workflows/other.yml b/.github/workflows/other.yml new file mode 100644 index 00000000000..d65b7abe575 --- /dev/null +++ b/.github/workflows/other.yml @@ -0,0 +1,138 @@ +name: Build and test (other) + +on: + push: + pull_request: + schedule: + # run daily, this refreshes the cache + - cron: "13 2 * * *" + +concurrency: # On new push, cancel old workflows from the same PR, branch or tag: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + python-test: + name: Python tests + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + python-version: ["2.7", "3.11"] + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # To check which files changed: origin/master..HEAD + - uses: LizardByte/setup-python-action@master + with: + python-version: ${{matrix.python-version}} + + - uses: actions/cache@v4 + name: Setup cache for running pre-commit fast + with: + path: ~/.cache/pre-commit + key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} + + - run: echo "::add-matcher::.github/workflows/python-warning-matcher.json" + name: "Setup GitHub for reporting Python warnings as annotations in pull request code review" + + - uses: pre-commit/action@v3.0.1 + name: Run pre-commit checks (no spaces at end of lines, etc) + if: ${{ matrix.python-version != '2.7' }} + with: + extra_args: --all-files --verbose --hook-stage commit + env: + SKIP: no-commit-to-branch + + - name: Install dependencies only needed for python 2 + if: ${{ matrix.python-version == '2.7' }} + run: pip install enum + + - name: Install dependencies only needed for python 3 + if: ${{ matrix.python-version != '2.7' }} + run: pip install opentelemetry-api opentelemetry-exporter-zipkin-json opentelemetry-sdk pandas pytype toml wrapt + + - name: Install common dependencies for Python ${{matrix.python-version}} + run: pip install future mock pytest-coverage pytest-mock + + - name: Run Pytest for python 2 and get code coverage for Codecov + if: ${{ matrix.python-version == '2.7' }} + run: > + pytest + --cov=scripts --cov=ocaml/xcp-rrdd + scripts/ ocaml/xcp-rrdd -vv -rA + --junitxml=.git/pytest${{matrix.python-version}}.xml + --cov-report term-missing + --cov-report xml:.git/coverage${{matrix.python-version}}.xml + env: + PYTHONDEVMODE: yes + + - name: Run Pytest for python 3 and get code coverage for Codecov + if: ${{ matrix.python-version != '2.7' }} + run: > + pytest + --cov=scripts --cov=ocaml/xcp-rrdd --cov=python3/ + scripts/ ocaml/xcp-rrdd python3/ -vv -rA + --junitxml=.git/pytest${{matrix.python-version}}.xml + --cov-report term-missing + --cov-report xml:.git/coverage${{matrix.python-version}}.xml + env: + PYTHONDEVMODE: yes + + - name: Upload Python ${{matrix.python-version}} coverage report to Codecov + uses: codecov/codecov-action@v3 + with: + directory: .git + files: coverage${{matrix.python-version}}.xml + env_vars: OS,PYTHON + fail_ci_if_error: false + flags: python${{matrix.python-version}} + name: coverage${{matrix.python-version}} + verbose: true + + - uses: dciborow/action-pylint@0.1.0 + if: ${{ matrix.python-version != '2.7' }} + with: + reporter: github-pr-review + level: warning + # To be customized to cover remaining Python scripts: + glob_pattern: "**/*.py" + continue-on-error: true + + - name: Run pytype checks + if: ${{ matrix.python-version != '2.7' }} + run: ./pytype_reporter.py + env: + PR_NUMBER: ${{ github.event.number }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PYTYPE_REPORTER_DEBUG: True + + - name: pyflakes + uses: reviewdog/action-pyflakes@v1 + with: + github_token: ${{ secrets.github_token }} + continue-on-error: true + + deprecation-test: + name: Deprecation tests + runs-on: ubuntu-22.04 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Generate empty configuration for make to be happy + run: touch config.mk + + - name: quality-gate + run: make quality-gate + + test-sdk-builds: + name: Test SDK builds + uses: ./.github/workflows/generate-and-build-sdks.yml + with: + # Ensure you also update ocaml-tests + # when changing this value, to keep builds + # consistent + xapi_version: "v0.0.0"