From 5c095f6a523bf85dd3955d0b68e73cabd0aa9c6c Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 24 Mar 2022 18:28:12 +0300 Subject: [PATCH 1/9] Fixes package tests --- .github/workflows/package-tests.yml | 35 ++--------------------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index 1e261388..43f17097 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -9,39 +9,8 @@ on: inputs: project_version: description: "The version to be tested" - required: true jobs: - metadata: - runs-on: ubuntu-latest - outputs: - pg_versions: ${{ steps.generate-postgres.outputs.pg_versions }} - citus_version: ${{ steps.get-citus-version.outputs.citus_version }} - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 2 - - name: Package version - id: get-citus-version - run: | - echo ${{ github.event.inputs.project_version }} - echo "Version: ${{ github.event.inputs.project_version }}" - - name: Install dependencies - run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources - - name: Install python requirements - run: python -m pip install -r packaging_automation/requirements.txt - - name: generate postgres - id: generate-postgres - run: | - export PROJECT_VERSION="${{ github.event.inputs.project_version }}" - echo "project_version: ${{ github.event.inputs.project_version }}" - # To be able to test pipeline without triggering with project_version parameter using workflow_dispatch parameter, - # if workflow_dispatch parameter is empty, 10.2.1 parameter is set to execute pipeline. - [ -z ${PROJECT_VERSION} ] && export PROJECT_VERSION=10.2.1 - POSTGRES_VERSIONS=$(python -m packaging_automation.get_postgres_versions --project_version ${PROJECT_VERSION}) - echo "Postgres Version: ${POSTGRES_VERSIONS}" - echo "::set-output name=pg_versions::${POSTGRES_VERSIONS}" test_execution: runs-on: ubuntu-latest needs: metadata @@ -59,7 +28,7 @@ jobs: - ubuntu/focal pg: ${{ fromJson(needs.metadata.outputs.pg_versions) }} env: - PLATFORM: ${{ matrix.platform }} + PLATFORM: [13, 14] steps: - name: Checkout repository @@ -77,6 +46,6 @@ jobs: echo "Citus Version: ${PROJECT_VERSION} " [ -z ${PROJECT_VERSION} ] && export PROJECT_VERSION=10.2.1 python -m packaging_automation.test_citus_package \ - --project_version "${PROJECT_VERSION}" \ + --project_version 11.0.0_beta \ --os_release ${{ matrix.platform }} \ --pg_major_version ${{ matrix.pg }} From 867584cce77b85be1dbd3ef2a8a77d323dd1ab22 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 24 Mar 2022 18:30:47 +0300 Subject: [PATCH 2/9] Fixes package tests --- .github/workflows/package-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index 43f17097..a194a583 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -1,4 +1,4 @@ -name: citus-package-tests +name: citus-package-tests2 on: push: From 2f6d4df469df5e5153438d86e2ee4e2b38d9ae6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Thu, 24 Mar 2022 18:34:28 +0300 Subject: [PATCH 3/9] Update package-tests.yml --- .github/workflows/package-tests.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index a194a583..840eb046 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -13,7 +13,6 @@ on: jobs: test_execution: runs-on: ubuntu-latest - needs: metadata strategy: fail-fast: false matrix: @@ -26,9 +25,9 @@ jobs: - debian/bullseye - ubuntu/bionic - ubuntu/focal - pg: ${{ fromJson(needs.metadata.outputs.pg_versions) }} + pg: [13,14] env: - PLATFORM: [13, 14] + PLATFORM: ${{ matrix.platform }} steps: - name: Checkout repository From 7f7675e4704f2a7b7d81b66b032cabd687289226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Thu, 24 Mar 2022 18:34:56 +0300 Subject: [PATCH 4/9] Update package-tests.yml --- .github/workflows/package-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index 840eb046..dc85983e 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -1,4 +1,4 @@ -name: citus-package-tests2 +name: citus-package-tests on: push: From c09d62e7fe23ee2c158d692b3ea190f19f6391e9 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 24 Mar 2022 18:41:04 +0300 Subject: [PATCH 5/9] Fixes package tests --- .github/workflows/package-tests.yml | 37 ++++++++++++++++++-- .github/workflows/package-tests2.yml | 51 ++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/package-tests2.yml diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index a194a583..1e261388 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -1,4 +1,4 @@ -name: citus-package-tests2 +name: citus-package-tests on: push: @@ -9,8 +9,39 @@ on: inputs: project_version: description: "The version to be tested" + required: true jobs: + metadata: + runs-on: ubuntu-latest + outputs: + pg_versions: ${{ steps.generate-postgres.outputs.pg_versions }} + citus_version: ${{ steps.get-citus-version.outputs.citus_version }} + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 2 + - name: Package version + id: get-citus-version + run: | + echo ${{ github.event.inputs.project_version }} + echo "Version: ${{ github.event.inputs.project_version }}" + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources + - name: Install python requirements + run: python -m pip install -r packaging_automation/requirements.txt + - name: generate postgres + id: generate-postgres + run: | + export PROJECT_VERSION="${{ github.event.inputs.project_version }}" + echo "project_version: ${{ github.event.inputs.project_version }}" + # To be able to test pipeline without triggering with project_version parameter using workflow_dispatch parameter, + # if workflow_dispatch parameter is empty, 10.2.1 parameter is set to execute pipeline. + [ -z ${PROJECT_VERSION} ] && export PROJECT_VERSION=10.2.1 + POSTGRES_VERSIONS=$(python -m packaging_automation.get_postgres_versions --project_version ${PROJECT_VERSION}) + echo "Postgres Version: ${POSTGRES_VERSIONS}" + echo "::set-output name=pg_versions::${POSTGRES_VERSIONS}" test_execution: runs-on: ubuntu-latest needs: metadata @@ -28,7 +59,7 @@ jobs: - ubuntu/focal pg: ${{ fromJson(needs.metadata.outputs.pg_versions) }} env: - PLATFORM: [13, 14] + PLATFORM: ${{ matrix.platform }} steps: - name: Checkout repository @@ -46,6 +77,6 @@ jobs: echo "Citus Version: ${PROJECT_VERSION} " [ -z ${PROJECT_VERSION} ] && export PROJECT_VERSION=10.2.1 python -m packaging_automation.test_citus_package \ - --project_version 11.0.0_beta \ + --project_version "${PROJECT_VERSION}" \ --os_release ${{ matrix.platform }} \ --pg_major_version ${{ matrix.pg }} diff --git a/.github/workflows/package-tests2.yml b/.github/workflows/package-tests2.yml new file mode 100644 index 00000000..a194a583 --- /dev/null +++ b/.github/workflows/package-tests2.yml @@ -0,0 +1,51 @@ +name: citus-package-tests2 + +on: + push: + branches: + - "**" + + workflow_dispatch: + inputs: + project_version: + description: "The version to be tested" + +jobs: + test_execution: + runs-on: ubuntu-latest + needs: metadata + strategy: + fail-fast: false + matrix: + platform: + - centos/8 + - centos/7 + - ol/7 + - debian/stretch + - debian/buster + - debian/bullseye + - ubuntu/bionic + - ubuntu/focal + pg: ${{ fromJson(needs.metadata.outputs.pg_versions) }} + env: + PLATFORM: [13, 14] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources + + - name: Install python requirements + run: python -m pip install -r packaging_automation/requirements.txt + + - name: Citus package tests + run: | + export PROJECT_VERSION="${{ github.event.inputs.project_version }}" + echo "Citus Version: ${PROJECT_VERSION} " + [ -z ${PROJECT_VERSION} ] && export PROJECT_VERSION=10.2.1 + python -m packaging_automation.test_citus_package \ + --project_version 11.0.0_beta \ + --os_release ${{ matrix.platform }} \ + --pg_major_version ${{ matrix.pg }} From a66b7e53a2574f93b649300a8c5d952dc6387ed4 Mon Sep 17 00:00:00 2001 From: gindibay Date: Thu, 24 Mar 2022 18:44:25 +0300 Subject: [PATCH 6/9] Fixes package tests --- .github/workflows/package-tests2.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/package-tests2.yml b/.github/workflows/package-tests2.yml index a194a583..aa1533df 100644 --- a/.github/workflows/package-tests2.yml +++ b/.github/workflows/package-tests2.yml @@ -5,15 +5,9 @@ on: branches: - "**" - workflow_dispatch: - inputs: - project_version: - description: "The version to be tested" - jobs: test_execution: runs-on: ubuntu-latest - needs: metadata strategy: fail-fast: false matrix: @@ -26,9 +20,9 @@ jobs: - debian/bullseye - ubuntu/bionic - ubuntu/focal - pg: ${{ fromJson(needs.metadata.outputs.pg_versions) }} + pg: [13, 14] env: - PLATFORM: [13, 14] + PLATFORM: ${{ matrix.platform }} steps: - name: Checkout repository From a71517d8f0f18f4506e968553a45f594d6e4c827 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Thu, 24 Mar 2022 18:46:22 +0300 Subject: [PATCH 7/9] retrt erere --- .github/workflows/package-tests2.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/package-tests2.yml b/.github/workflows/package-tests2.yml index aa1533df..2b9d4e95 100644 --- a/.github/workflows/package-tests2.yml +++ b/.github/workflows/package-tests2.yml @@ -6,6 +6,7 @@ on: - "**" jobs: + test_execution: runs-on: ubuntu-latest strategy: From 8f393fd2890322f0987fdedf076beaf0761e0b76 Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 29 Mar 2022 14:14:50 +0300 Subject: [PATCH 8/9] Fixes beta key in version error --- .github/workflows/package-tests.yml | 3 ++- packaging_automation/common_tool_methods.py | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index 6577a5cf..1e261388 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -44,6 +44,7 @@ jobs: echo "::set-output name=pg_versions::${POSTGRES_VERSIONS}" test_execution: runs-on: ubuntu-latest + needs: metadata strategy: fail-fast: false matrix: @@ -56,7 +57,7 @@ jobs: - debian/bullseye - ubuntu/bionic - ubuntu/focal - pg: [13,14] + pg: ${{ fromJson(needs.metadata.outputs.pg_versions) }} env: PLATFORM: ${{ matrix.platform }} diff --git a/packaging_automation/common_tool_methods.py b/packaging_automation/common_tool_methods.py index 55b8167d..4f290b81 100644 --- a/packaging_automation/common_tool_methods.py +++ b/packaging_automation/common_tool_methods.py @@ -20,6 +20,8 @@ from .common_validations import (is_tag, is_version) from .dbconfig import RequestLog, RequestType +RELEASE_STAGE_SEPARATOR = "_" + BASE_GIT_PATH = pathlib2.Path(__file__).parents[1] PATCH_VERSION_MATCH_FROM_MINOR_SUFFIX = r"\.\d{1,3}" POSTGRES_MATRIX_FLIE_NAME = "postgres-matrix.yml" @@ -87,7 +89,7 @@ def get_minor_project_version(project_version: str) -> str: def project_version_contains_release_stage(project_version: str) -> bool: - return "_" in project_version + return RELEASE_STAGE_SEPARATOR in project_version def get_minor_project_version_for_docker(project_version: str) -> str: @@ -395,7 +397,7 @@ def remove_cloned_code(exec_path: str): def process_template_file_with_minor(project_version: str, templates_path: str, template_file_path: str, - minor_version: str,postgres_version: str = ""): + minor_version: str, postgres_version: str = ""): ''' This function gets the template files, changes tha parameters inside the file and returns the output. Template files are stored under packaging_automation/templates and these files include parametric items in the format of {{parameter_name}}. This function is used while creating docker files and pgxn files which include @@ -652,6 +654,9 @@ def match_release_version(versions_dictionary, package_version: str): def get_numeric_counterpart_of_version(package_version: str): + # If there is release stage + if project_version_contains_release_stage(project_version=package_version): + package_version = package_version.split(RELEASE_STAGE_SEPARATOR)[0] numbers_in_version = package_version.split(".") # add a 0 if version is minor to calculate and match for patch releases accurately if len(numbers_in_version) == 2: From cd3cbd0278091d7bee9d01ae6d9a031071650dbc Mon Sep 17 00:00:00 2001 From: gindibay Date: Tue, 29 Mar 2022 14:38:39 +0300 Subject: [PATCH 9/9] Removes temp pipeline --- .github/workflows/package-tests2.yml | 46 ---------------------------- 1 file changed, 46 deletions(-) delete mode 100644 .github/workflows/package-tests2.yml diff --git a/.github/workflows/package-tests2.yml b/.github/workflows/package-tests2.yml deleted file mode 100644 index 2b9d4e95..00000000 --- a/.github/workflows/package-tests2.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: citus-package-tests2 - -on: - push: - branches: - - "**" - -jobs: - - test_execution: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - platform: - - centos/8 - - centos/7 - - ol/7 - - debian/stretch - - debian/buster - - debian/bullseye - - ubuntu/bionic - - ubuntu/focal - pg: [13, 14] - env: - PLATFORM: ${{ matrix.platform }} - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Install dependencies - run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources - - - name: Install python requirements - run: python -m pip install -r packaging_automation/requirements.txt - - - name: Citus package tests - run: | - export PROJECT_VERSION="${{ github.event.inputs.project_version }}" - echo "Citus Version: ${PROJECT_VERSION} " - [ -z ${PROJECT_VERSION} ] && export PROJECT_VERSION=10.2.1 - python -m packaging_automation.test_citus_package \ - --project_version 11.0.0_beta \ - --os_release ${{ matrix.platform }} \ - --pg_major_version ${{ matrix.pg }}