From c3640c82bc0f49b20db1f6bdc99dded99bcd1264 Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Wed, 17 Jun 2020 09:37:26 +0300 Subject: [PATCH 01/11] added rhel wagon, changed the wagon builders to py2py3 wagons and enable integration tests --- .circleci/config.yml | 53 +++++++++++++++++++++++++++++++++++--------- CHANGELOG.txt | 3 +++ plugin.yaml | 4 ++-- setup.py | 2 +- 4 files changed, 48 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e0cfc4b1..1169fca7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -74,16 +74,16 @@ commands: exit 1 fi - generate_py27_wagon: + generate_py27py36_wagon: steps: - run: name: Create Workspace Build directory. command: mkdir -p workspace/build - run: - name: Build py27 Wagon + name: Build py27py36 Wagon command: | git clone https://github.com/cloudify-cosmo/cloudify-wagon-build-containers.git - docker build -t cloudify-centos-7-wagon-builder cloudify-wagon-build-containers/centos_7 + docker build -t cloudify-centos-7-wagon-builder cloudify-wagon-build-containers/centos_7_py2py3 docker run -v ~/project/:/packaging cloudify-centos-7-wagon-builder - run: name: copy wagon to workspace @@ -93,6 +93,25 @@ commands: paths: - build/* + generate_rhel_py27py36_wagon: + steps: + - run: + name: Create Workspace Build directory. + command: mkdir -p workspace/build + - run: + name: Build RHEL py27py36 Wagon + command: | + git clone https://github.com/cloudify-cosmo/cloudify-wagon-build-containers.git + docker build -t cloudify-redhat-7-wagon-builder cloudify-wagon-build-containers/redhat_7_py2py3 --build-arg USERNAME="$USERNAME" --build-arg PASSWORD="$PASSWORD" + docker run -v ~/project/:/packaging cloudify-redhat-7-wagon-builder + - run: + name: copy wagon to workspace + command: cp *.wgn workspace/build/ + - persist_to_workspace: + root: workspace + paths: + - build/* + generate_py3_wagon: steps: - run: @@ -202,9 +221,15 @@ jobs: executor: wagon_generator steps: - checkout - - generate_py27_wagon + - generate_py27py36_wagon # - generate_py3_wagon # TODO: When we have Python3 Manager Integration Test. + rhel_wagon: + executor: wagon_generator + steps: + - checkout + - generate_rhel_py27py36_wagon + integration_tests: executor: cloudify-machine environment: @@ -227,22 +252,28 @@ workflows: - py3_compat - unittests_py27 - unittests_py36 - - wagon: - filters: - branches: - only: /([0-9\.]*\-build|master|dev)/ + - wagon #: +# filters: +# branches: +# only: /([0-9\.]*\-build|master|dev)/ + - rhel_wagon #: + # filters: + # branches: + # only: /([0-9\.]*\-build|master|dev)/ - integration_tests: requires: - wagon - filters: - branches: - only: /([0-9\.]*\-build|master|dev)/ + - rhel_wagon +# filters: +# branches: +# only: /([0-9\.]*\-build|master|dev)/ - release: filters: branches: only: /master/ requires: - wagon + - rhel_wagon - integration_tests nightly: triggers: diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 9db04f02..0b4b3d67 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -71,3 +71,6 @@ 2.9.1: - Fix Bug introduced by plugin yaml changes in 2.9.1 + +2.9.2: + - Update wagon builder to py2py3 wagon and add rhel wagon builder. \ No newline at end of file diff --git a/plugin.yaml b/plugin.yaml index 2fb03626..d1540737 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -2,9 +2,9 @@ plugins: ansible: executor: central_deployment_agent - source: https://github.com/cloudify-cosmo/cloudify-ansible-plugin/archive/2.9.1.zip + source: https://github.com/cloudify-cosmo/cloudify-ansible-plugin/archive/2.9.2.zip package_name: cloudify-ansible-plugin - package_version: '2.9.1' + package_version: '2.9.2' dsl_definitions: diff --git a/setup.py b/setup.py index 650141dd..2bae573e 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ setup( name='cloudify-ansible-plugin', - version='2.9.1', + version='2.9.2', author='Cloudify Platform LTD', author_email='hello@cloudify.co', description='Manage Ansible nodes by Cloudify.', From ceda1388d3fff3dd0bc4df7f1b4e036abb93a97a Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Wed, 17 Jun 2020 10:18:18 +0300 Subject: [PATCH 02/11] disable rhel wagon build in order to invoke integration tests --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1169fca7..f834a225 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -228,7 +228,7 @@ jobs: executor: wagon_generator steps: - checkout - - generate_rhel_py27py36_wagon +# - generate_rhel_py27py36_wagon integration_tests: executor: cloudify-machine From 498bbe50066825c8be6d920f56e0bc25dc26c740 Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Sun, 21 Jun 2020 14:33:44 +0300 Subject: [PATCH 03/11] enabled rhel wagon builder --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f834a225..1169fca7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -228,7 +228,7 @@ jobs: executor: wagon_generator steps: - checkout -# - generate_rhel_py27py36_wagon + - generate_rhel_py27py36_wagon integration_tests: executor: cloudify-machine From 98046f10200e027e7ccb7da2a0217f49db617763 Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Sun, 21 Jun 2020 15:48:46 +0300 Subject: [PATCH 04/11] return integration tests to normal branches --- .circleci/config.yml | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1169fca7..b11049d0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -222,7 +222,6 @@ jobs: steps: - checkout - generate_py27py36_wagon - # - generate_py3_wagon # TODO: When we have Python3 Manager Integration Test. rhel_wagon: executor: wagon_generator @@ -252,21 +251,21 @@ workflows: - py3_compat - unittests_py27 - unittests_py36 - - wagon #: -# filters: -# branches: -# only: /([0-9\.]*\-build|master|dev)/ - - rhel_wagon #: - # filters: - # branches: - # only: /([0-9\.]*\-build|master|dev)/ + - wagon: + filters: + branches: + only: /([0-9\.]*\-build|master|dev)/ + - rhel_wagon: + filters: + branches: + only: /([0-9\.]*\-build|master|dev)/ - integration_tests: requires: - wagon - rhel_wagon -# filters: -# branches: -# only: /([0-9\.]*\-build|master|dev)/ + filters: + branches: + only: /([0-9\.]*\-build|master|dev)/ - release: filters: branches: From dd6d4d3f2a04093cedf037796d5536ddc1181894 Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Sun, 21 Jun 2020 19:45:11 +0300 Subject: [PATCH 05/11] first try for 5.1 integration tests --- .circleci/config.yml | 66 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b11049d0..34f1958d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,6 +27,9 @@ executors: machine: image: ubuntu-1604:201903-01 + cloudify-machine-510: + machine: + image: ubuntu-1604:201903-01 commands: run_unittest: @@ -131,7 +134,7 @@ commands: paths: - build/* - run_integration_tests: + run_integration_tests_505: steps: - run: name: "Pull Submodules" @@ -165,6 +168,43 @@ commands: command: pip install https://github.com/cloudify-incubator/cloudify-ecosystem-test/archive/latest.zip - run: pytest -s .circleci/test_examples.py + run_integration_tests_510: + steps: + - run: + name: "Pull Submodules" + command: | + git submodule update --init --recursive + - attach_workspace: + at: workspace + - run: + name: update pip + command: pip install -U pip + - run: + name: install cloudify + command: > + pip install -U git+https://github.com/cloudify-cosmo/cloudify-common@master#egg=cloudify-common[dispatcher]==master + git+https://github.com/cloudify-cosmo/cloudify-cli@master#egg=cloudify-cli==master + pytest==4.6.3 pyyaml==3.12 + - run: + name: download manager docker image + command: wget http://repository.cloudifysource.org/cloudify/5.1.0/.dev1-release/cloudify-docker-manager-5.1.0.dev1.tar + - run: + name: load docker image + command: docker load -i cloudify-docker-manager-5.1.0.dev1.tar + - run: + name: retain space by dumping the tar + command: rm cloudify-docker-manager-5.1.0.dev1.tar + - run: + name: show images + command: docker images + - run: + name: start docker container + command: docker run --name cfy_manager -d --restart unless-stopped -v /sys/fs/cgroup:/sys/fs/cgroup:ro --tmpfs /run --tmpfs /run/lock --security-opt seccomp:unconfined --cap-add SYS_ADMIN --network host cloudifyplatform/premium-cloudify-manager-aio + - run: + name: install local project + command: pip install https://github.com/cloudify-incubator/cloudify-ecosystem-test/archive/latest.zip + - run: pytest -s .circleci/test_examples.py + release_plugin: steps: - run: @@ -229,15 +269,23 @@ jobs: - checkout - generate_rhel_py27py36_wagon - integration_tests: + integration_tests_505: executor: cloudify-machine environment: CLOUDIFY_SSL_TRUST_ALL: true IAAS: gcp steps: - checkout - - run_integration_tests + - run_integration_tests_505 + integration_tests_510: + executor: cloudify-machine-510 + environment: + CLOUDIFY_SSL_TRUST_ALL: true + IAAS: gcp + steps: + - checkout + - run_integration_tests_510 release: executor: py27 steps: @@ -259,13 +307,20 @@ workflows: filters: branches: only: /([0-9\.]*\-build|master|dev)/ - - integration_tests: + - integration_tests_505: requires: - wagon - rhel_wagon filters: branches: only: /([0-9\.]*\-build|master|dev)/ + - integration_tests_510: + requires: + - wagon + - rhel_wagon +# filters: +# branches: +# only: /([0-9\.]*\-build|master|dev)/ - release: filters: branches: @@ -273,7 +328,8 @@ workflows: requires: - wagon - rhel_wagon - - integration_tests + - integration_tests_505 + - integration_tests_510 nightly: triggers: - schedule: From 619a362227b00fd187570a87fd114f7a3e45909c Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Sun, 21 Jun 2020 19:48:01 +0300 Subject: [PATCH 06/11] enable wagon builders on branch --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 34f1958d..527d8026 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -302,11 +302,11 @@ workflows: - wagon: filters: branches: - only: /([0-9\.]*\-build|master|dev)/ + only: /([0-9\.]*\-build|master|dev|replace-wagon-builder-to-py2py3-builder)/ - rhel_wagon: filters: branches: - only: /([0-9\.]*\-build|master|dev)/ + only: /([0-9\.]*\-build|master|dev|replace-wagon-builder-to-py2py3-builder)/ - integration_tests_505: requires: - wagon From c4e664827e113744198f9307d72322df537b3dc2 Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Sat, 27 Jun 2020 14:04:12 +0300 Subject: [PATCH 07/11] added 5.1 integration tests --- .circleci/config.yml | 65 ++++++++++++++++++++------------------ .circleci/test_examples.py | 12 ++++++- CHANGELOG.txt | 3 +- test-requirements.txt | 7 +++- 4 files changed, 54 insertions(+), 33 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 527d8026..7b814ba6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -115,25 +115,6 @@ commands: paths: - build/* - generate_py3_wagon: - steps: - - run: - name: Create Workspace Build directory. - command: mkdir -p workspace/build - - run: - name: Build py3 Wagon - command: | - git clone https://github.com/cloudify-cosmo/cloudify-wagon-build-containers.git - docker build -t cloudify-centos-7-py3-wagon-builder cloudify-wagon-build-containers/centos_7_py3 - docker run -v ~/project/:/packaging cloudify-centos-7-py3-wagon-builder - - run: - name: copy wagon to workspace - command: cp *.wgn workspace/build/ - - persist_to_workspace: - root: workspace - paths: - - build/* - run_integration_tests_505: steps: - run: @@ -147,7 +128,8 @@ commands: command: pip install -U pip - run: name: install cloudify - command: pip install -U cloudify==5.0.0 pytest==4.6.3 pyyaml==3.10 + command: | + pip install -r test-requirements.txt - run: name: download manager docker image command: wget http://repository.cloudifysource.org/cloudify/5.0.5/ga-release/cloudify-docker-manager-5.0.5.tar @@ -181,10 +163,8 @@ commands: command: pip install -U pip - run: name: install cloudify - command: > - pip install -U git+https://github.com/cloudify-cosmo/cloudify-common@master#egg=cloudify-common[dispatcher]==master - git+https://github.com/cloudify-cosmo/cloudify-cli@master#egg=cloudify-cli==master - pytest==4.6.3 pyyaml==3.12 + command: | + pip install -r test-requirements.txt - run: name: download manager docker image command: wget http://repository.cloudifysource.org/cloudify/5.1.0/.dev1-release/cloudify-docker-manager-5.1.0.dev1.tar @@ -295,7 +275,7 @@ jobs: workflows: version: 2 tests: - jobs: &all_jobs + jobs: - py3_compat - unittests_py27 - unittests_py36 @@ -313,14 +293,14 @@ workflows: - rhel_wagon filters: branches: - only: /([0-9\.]*\-build|master|dev)/ + only: /([0-9\.]*\-build|master|dev|replace-wagon-builder-to-py2py3-builder)/ - integration_tests_510: requires: - wagon - rhel_wagon -# filters: -# branches: -# only: /([0-9\.]*\-build|master|dev)/ + filters: + branches: + only: /([0-9\.]*\-build|master|dev|replace-wagon-builder-to-py2py3-builder)/ - release: filters: branches: @@ -338,4 +318,29 @@ workflows: branches: only: - master - jobs: *all_jobs + jobs: + - py3_compat + - unittests_py27 + - unittests_py36 + - wagon: + filters: + branches: + only: /([0-9\.]*\-build|master|dev)/ + - rhel_wagon: + filters: + branches: + only: /([0-9\.]*\-build|master|dev)/ + - integration_tests_505: + requires: + - wagon + - rhel_wagon + filters: + branches: + only: /([0-9\.]*\-build|master|dev)/ + - integration_tests_510: + requires: + - wagon + - rhel_wagon + filters: + branches: + only: /([0-9\.]*\-build|master|dev)/ \ No newline at end of file diff --git a/.circleci/test_examples.py b/.circleci/test_examples.py index acd10a86..4cccaec9 100644 --- a/.circleci/test_examples.py +++ b/.circleci/test_examples.py @@ -22,11 +22,21 @@ cleanup_on_failure, prepare_test ) +'''Temporary until all the plugins in the bundle will +released with py2py3 wagons''' +UT_VERSION = '1.23.5' +UT_WAGON = 'https://github.com/cloudify-incubator/cloudify-utilities-plugin/' \ + 'releases/download/{v}/cloudify_utilities_plugin-{v}-centos' \ + '-Core-py27.py36-none-linux_x86_64.wgn'.format(v=UT_VERSION) +UT_PLUGIN = 'https://github.com/cloudify-incubator/cloudify-utilities-' \ + 'plugin/releases/download/{v}/plugin.yaml'.format(v=UT_VERSION) +PLUGINS_TO_UPLOAD = [(UT_WAGON, UT_PLUGIN)] SECRETS_TO_CREATE = { 'gcp_credentials': True } -prepare_test(secrets=SECRETS_TO_CREATE) +prepare_test(plugins=PLUGINS_TO_UPLOAD, secrets=SECRETS_TO_CREATE, + execute_bundle_upload=False) blueprint_list = ['examples/blueprint-examples/hello-world-example/gcp.yaml'] diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 0b4b3d67..3733ae1c 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -73,4 +73,5 @@ - Fix Bug introduced by plugin yaml changes in 2.9.1 2.9.2: - - Update wagon builder to py2py3 wagon and add rhel wagon builder. \ No newline at end of file + - Update wagon builder to py2py3 wagon and add rhel wagon builder. + - Added 5.1.0 manager integration tests. \ No newline at end of file diff --git a/test-requirements.txt b/test-requirements.txt index a3ceb07d..cb4b5837 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,8 @@ -https://github.com/cloudify-cosmo/cloudify-common/archive/master.zip#egg=cloudify[dispatcher] nose mock + +# For integration tests +git+https://github.com/cloudify-cosmo/cloudify-common@master#egg=cloudify-common[dispatcher]==master +git+https://github.com/cloudify-cosmo/cloudify-cli@master#egg=cloudify-cli==master +pytest==4.6.3 +pyyaml==3.12 \ No newline at end of file From 120e2f68282b4275b87e6da3b2a630dfecb70571 Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Sat, 27 Jun 2020 14:37:43 +0300 Subject: [PATCH 08/11] added unitests to release requirements --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7b814ba6..104f0bd8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -306,6 +306,8 @@ workflows: branches: only: /master/ requires: + - unittests_py27 + - unittests_py36 - wagon - rhel_wagon - integration_tests_505 From 70ab4a962239dab7b217d4ac75014ee3eca318dc Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Sun, 28 Jun 2020 13:12:29 +0300 Subject: [PATCH 09/11] added gcp to plugins upload list --- .circleci/test_examples.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.circleci/test_examples.py b/.circleci/test_examples.py index 4cccaec9..abd955e4 100644 --- a/.circleci/test_examples.py +++ b/.circleci/test_examples.py @@ -30,7 +30,13 @@ '-Core-py27.py36-none-linux_x86_64.wgn'.format(v=UT_VERSION) UT_PLUGIN = 'https://github.com/cloudify-incubator/cloudify-utilities-' \ 'plugin/releases/download/{v}/plugin.yaml'.format(v=UT_VERSION) -PLUGINS_TO_UPLOAD = [(UT_WAGON, UT_PLUGIN)] +GCP_VERSION = '1.6.6' +GCP_WAGON = 'https://github.com/cloudify-cosmo/cloudify-gcp-plugin/' \ + 'releases/download/{v}/cloudify_gcp_plugin-{v}-centos-' \ + 'Core-py27.py36-none-linux_x86_64.wgn'.format(v=GCP_VERSION) +GCP_PLUGIN = 'https://github.com/cloudify-cosmo/cloudify-gcp-plugin/releases/' \ + 'download/{v}/plugin.yaml'.format(v=GCP_VERSION) +PLUGINS_TO_UPLOAD = [(UT_WAGON, UT_PLUGIN), (GCP_WAGON, GCP_PLUGIN)] SECRETS_TO_CREATE = { 'gcp_credentials': True } From 225443241d747f37250b2e434d153d5121809c9c Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Sun, 28 Jun 2020 13:37:57 +0300 Subject: [PATCH 10/11] return integration tests filter of branches to normal --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 104f0bd8..4f8f2a45 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -282,25 +282,25 @@ workflows: - wagon: filters: branches: - only: /([0-9\.]*\-build|master|dev|replace-wagon-builder-to-py2py3-builder)/ + only: /([0-9\.]*\-build|master|dev)/ - rhel_wagon: filters: branches: - only: /([0-9\.]*\-build|master|dev|replace-wagon-builder-to-py2py3-builder)/ + only: /([0-9\.]*\-build|master|dev)/ - integration_tests_505: requires: - wagon - rhel_wagon filters: branches: - only: /([0-9\.]*\-build|master|dev|replace-wagon-builder-to-py2py3-builder)/ + only: /([0-9\.]*\-build|master|dev)/ - integration_tests_510: requires: - wagon - rhel_wagon filters: branches: - only: /([0-9\.]*\-build|master|dev|replace-wagon-builder-to-py2py3-builder)/ + only: /([0-9\.]*\-build|master|dev)/ - release: filters: branches: From a377d30c72c00e12e690063d58248189b81741ed Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Sun, 28 Jun 2020 13:40:16 +0300 Subject: [PATCH 11/11] added newline at end of config file --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4f8f2a45..9fdaec67 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -345,4 +345,5 @@ workflows: - rhel_wagon filters: branches: - only: /([0-9\.]*\-build|master|dev)/ \ No newline at end of file + only: /([0-9\.]*\-build|master|dev)/ + \ No newline at end of file