From 5a3fc637bf82347b40e9d432cf229deafa1926b4 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Wed, 30 Aug 2023 15:15:56 +0300 Subject: [PATCH 1/6] Put support templates into a cloud agnostic location --- .pre-commit-config.yaml | 2 +- .../templates/{gcp => common}/support.secret.values.yaml | 0 config/clusters/templates/{gcp => common}/support.values.yaml | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename config/clusters/templates/{gcp => common}/support.secret.values.yaml (100%) rename config/clusters/templates/{gcp => common}/support.values.yaml (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f2749233..824d8d8bd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,7 +53,7 @@ repos: hooks: - id: sops-encryption # Add files here if they contain the word 'secret' but should not be encrypted - exclude: secrets\.md|helm-charts/support/templates/prometheus-ingres-auth/secret\.yaml|helm-charts/basehub/templates/dex/secret\.yaml|helm-charts/basehub/templates/static/secret\.yaml|config/clusters/templates/gcp/support\.secret\.values\.yaml|helm-charts/basehub/templates/ingress-auth/secret\.yaml + exclude: secrets\.md|helm-charts/support/templates/prometheus-ingres-auth/secret\.yaml|helm-charts/basehub/templates/dex/secret\.yaml|helm-charts/basehub/templates/static/secret\.yaml|config/clusters/templates/common/support\.secret\.values\.yaml|helm-charts/basehub/templates/ingress-auth/secret\.yaml # pre-commit.ci config reference: https://pre-commit.ci/#configuration ci: diff --git a/config/clusters/templates/gcp/support.secret.values.yaml b/config/clusters/templates/common/support.secret.values.yaml similarity index 100% rename from config/clusters/templates/gcp/support.secret.values.yaml rename to config/clusters/templates/common/support.secret.values.yaml diff --git a/config/clusters/templates/gcp/support.values.yaml b/config/clusters/templates/common/support.values.yaml similarity index 100% rename from config/clusters/templates/gcp/support.values.yaml rename to config/clusters/templates/common/support.values.yaml From 1f21f27b1dc8c9b8ea537c35007556c4a0ae85ec Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Wed, 30 Aug 2023 15:17:03 +0300 Subject: [PATCH 2/6] Put the common generation config into its own module --- deployer/generate/common.py | 95 +++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 deployer/generate/common.py diff --git a/deployer/generate/common.py b/deployer/generate/common.py new file mode 100644 index 000000000..702b201ee --- /dev/null +++ b/deployer/generate/common.py @@ -0,0 +1,95 @@ +import os +import secrets +import string +import subprocess +from pathlib import Path + +import jinja2 + +from ..utils import print_colour + +REPO_ROOT = Path(__file__).parent.parent.parent + + +def generate_cluster_config_file(cluster_config_directory, vars): + """ + Generates the `config//cluster.yaml` config + """ + with open(REPO_ROOT / "config/clusters/templates/gcp/cluster.yaml") as f: + cluster_yaml_template = jinja2.Template(f.read()) + with open(cluster_config_directory / "cluster.yaml", "w") as f: + f.write(cluster_yaml_template.render(**vars)) + + +def generate_support_files(cluster_config_directory, vars): + """ + Generates files related to support components. + + They are required to deploy the support chart for the cluster + and to configure the Prometheus instance. + + Generates: + - `config//support.values.yaml` + - `config//enc-support.secret.values.yaml` + """ + # Generate the suppport values file `support.values.yaml` + print_colour("Generating the support values file...", "yellow") + with open(REPO_ROOT / "config/clusters/templates/common/support.values.yaml") as f: + support_values_yaml_template = jinja2.Template(f.read()) + + with open(cluster_config_directory / "support.values.yaml", "w") as f: + f.write(support_values_yaml_template.render(**vars)) + print_colour(f"{cluster_config_directory}/support.values.yaml created") + + # Generate and encrypt prometheus credentials into `enc-support.secret.values.yaml` + print_colour("Generating the prometheus credentials encrypted file...", "yellow") + alphabet = string.ascii_letters + string.digits + credentials = { + "username": "".join(secrets.choice(alphabet) for i in range(64)), + "password": "".join(secrets.choice(alphabet) for i in range(64)), + } + with open( + REPO_ROOT / "config/clusters/templates/common/support.secret.values.yaml" + ) as f: + support_secret_values_yaml_template = jinja2.Template(f.read()) + with open(cluster_config_directory / "enc-support.secret.values.yaml", "w") as f: + f.write(support_secret_values_yaml_template.render(**credentials)) + + # Encrypt the private key + subprocess.check_call( + [ + "sops", + "--in-place", + "--encrypt", + cluster_config_directory / "enc-support.secret.values.yaml", + ] + ) + print_colour( + f"{cluster_config_directory}/enc-support.values.yaml created and encrypted" + ) + + +def generate_config_directory(vars): + """ + Generates the required `config` directory for hubs on a GCP cluster + + Generates the following files: + - `config//cluster.yaml` + - `config//support.values.yaml` + - `config//enc-support.secret.values.yaml` + """ + cluster_config_directory = REPO_ROOT / "config/clusters" / vars["cluster_name"] + + print_colour( + f"Checking if cluster config directory {cluster_config_directory} exists...", + "yellow", + ) + if os.path.exists(cluster_config_directory): + print_colour(f"{cluster_config_directory} already exists.") + return cluster_config_directory + + # Create the cluster config directory and initial `cluster.yaml` file + os.makedirs(cluster_config_directory) + print_colour(f"{cluster_config_directory} created") + + return cluster_config_directory From 82d38eb33c389491fa3cf057010367ba086dca22 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Wed, 30 Aug 2023 15:17:33 +0300 Subject: [PATCH 3/6] Have the gcp generator use the common module --- deployer/generate/generate_gcp_cluster.py | 158 ++++++---------------- 1 file changed, 41 insertions(+), 117 deletions(-) diff --git a/deployer/generate/generate_gcp_cluster.py b/deployer/generate/generate_gcp_cluster.py index 208ce501b..371451348 100644 --- a/deployer/generate/generate_gcp_cluster.py +++ b/deployer/generate/generate_gcp_cluster.py @@ -1,111 +1,64 @@ -import os -import secrets -import string -import subprocess -from pathlib import Path +""" +Generates the ` terraform file required to create a GCP cluster +and the required `config` directory for hubs on a GCP cluster. +Generates the following files: +- terraform/gcp/projects/.tfvars` +- `config//cluster.yaml` +- `config//support.values.yaml` +- `config//enc-support.secret.values.yaml` + +""" import jinja2 import typer from ..cli_app import app from ..utils import print_colour - -REPO_ROOT = Path(__file__).parent.parent.parent +from .common import ( + REPO_ROOT, + generate_cluster_config_file, + generate_config_directory, + generate_support_files, +) -def generate_terraform_file(cluster_name, cluster_region, project_id, hub_type): +def generate_terraform_file(vars): """ Generates the `terraform/gcp/projects/.tfvars` terraform file required to create a GCP cluster """ - with open(REPO_ROOT / f"terraform/gcp/projects/{hub_type}-template.tfvars") as f: - tfvars_template = jinja2.Template(f.read()) - - vars = { - "cluster_name": cluster_name, - "cluster_region": cluster_region, - "project_id": project_id, - } - - print_colour("Generating the terraform infrastructure file...", "yellow") with open( - REPO_ROOT / "terraform/gcp/projects" / f"{cluster_name}.tfvars", "w" + REPO_ROOT / f'terraform/gcp/projects/{vars["hub_type"]}-template.tfvars' ) as f: - f.write(tfvars_template.render(**vars)) - print_colour(f"{REPO_ROOT}/terraform/gcp/projects/{cluster_name}.tfvars created") - - -def generate_cluster_config_file(cluster_config_directory, vars): - """ - Generates the `config//cluster.yaml` config - """ - with open(REPO_ROOT / "config/clusters/templates/gcp/cluster.yaml") as f: - cluster_yaml_template = jinja2.Template(f.read()) - with open(cluster_config_directory / "cluster.yaml", "w") as f: - f.write(cluster_yaml_template.render(**vars)) - - -def generate_support_files(cluster_config_directory, vars): - """ - Generates files related to support components. - - They are required to deploy the support chart for the cluster - and to configure the Prometheus instance. - - Generates: - - `config//support.values.yaml` - - `config//enc-support.secret.values.yaml` - """ - # Generate the suppport values file `support.values.yaml` - print_colour("Generating the support values file...", "yellow") - with open(REPO_ROOT / "config/clusters/templates/gcp/support.values.yaml") as f: - support_values_yaml_template = jinja2.Template(f.read()) - - with open(cluster_config_directory / "support.values.yaml", "w") as f: - f.write(support_values_yaml_template.render(**vars)) - print_colour(f"{cluster_config_directory}/support.values.yaml created") - - # Generate and encrypt prometheus credentials into `enc-support.secret.values.yaml` - print_colour("Generating the prometheus credentials encrypted file...", "yellow") - alphabet = string.ascii_letters + string.digits + string.punctuation - credentials = { - "username": "".join(secrets.choice(alphabet) for i in range(64)), - "password": "".join(secrets.choice(alphabet) for i in range(64)), - } - with open( - REPO_ROOT / "config/clusters/templates/gcp/support.secret.values.yaml" - ) as f: - support_secret_values_yaml_template = jinja2.Template(f.read()) - with open(cluster_config_directory / "enc-support.secret.values.yaml", "w") as f: - f.write(support_secret_values_yaml_template.render(**credentials)) + tfvars_template = jinja2.Template(f.read()) - # Encrypt the private key - subprocess.check_call( - [ - "sops", - "--in-place", - "--encrypt", - cluster_config_directory / "enc-support.secret.values.yaml", - ] - ) - print_colour( - f"{cluster_config_directory}/enc-support.values.yaml created and encrypted" + print_colour("Generating the terraform infrastructure file...", "yellow") + tfvars_file_path = ( + REPO_ROOT / "terraform/gcp/projects" / f'{vars["cluster_name"]}.tfvars' ) + with open(tfvars_file_path, "w") as f: + f.write(tfvars_template.render(**vars)) + print_colour(f"{tfvars_file_path} created") -def generate_config_directory( - cluster_name, cluster_region, project_id, hub_type, hub_name +@app.command() +def generate_gcp_cluster( + cluster_name: str = typer.Option(..., prompt="Name of the cluster"), + cluster_region: str = typer.Option(..., prompt="Cluster region"), + project_id: str = typer.Option(..., prompt="Project ID of the GCP project"), + hub_type: str = typer.Option( + ..., prompt="Type of hub. Choose from `basehub` or `daskhub`" + ), + hub_name: str = typer.Option(..., prompt="Name of the first hub"), ): """ - Generates the required `config` directory for hubs on a GCP cluster - - Generates the following files: - - `config//cluster.yaml` - - `config//support.values.yaml` - - `config//enc-support.secret.values.yaml` + Automatically generates the initial files, required to setup a new cluster on GCP """ - cluster_config_directory = REPO_ROOT / "config/clusters" / cluster_name + # Automatically generate the terraform config file + generate_terraform_file(cluster_name, cluster_region, project_id, hub_type) + # These are the variables needed by the templates used to generate the cluster config file + # and support files vars = { "cluster_name": cluster_name, "hub_type": hub_type, @@ -114,40 +67,11 @@ def generate_config_directory( "hub_name": hub_name, } - print_colour( - "Checking if cluster config directory {cluster_config_directory} exists...", - "yellow", - ) - if os.path.exists(cluster_config_directory): - print_colour(f"{cluster_config_directory} already exists.") - return + # Automatically generate the config directory + cluster_config_directory = generate_config_directory(vars) # Create the cluster config directory and initial `cluster.yaml` file - os.makedirs(cluster_config_directory) - print_colour(f"{cluster_config_directory} created") generate_cluster_config_file(cluster_config_directory, vars) # Generate the support files generate_support_files(cluster_config_directory, vars) - - -@app.command() -def generate_gcp_cluster( - cluster_name: str = typer.Option(..., prompt="Name of the cluster"), - cluster_region: str = typer.Option(..., prompt="Cluster region"), - project_id: str = typer.Option(..., prompt="Project ID of the GCP project"), - hub_type: str = typer.Option( - ..., prompt="Type of hub. Choose from `basehub` or `daskhub`" - ), - hub_name: str = typer.Option(..., prompt="Name of the first hub"), -): - """ - Automatically generates the initial files, required to setup a new cluster on GCP - """ - # Automatically generate the terraform config file - generate_terraform_file(cluster_name, cluster_region, project_id, hub_type) - - # Automatically generate the config directory - generate_config_directory( - cluster_name, cluster_region, project_id, hub_type, hub_name - ) From b80997964cc80d2caaf47634fe9e8b8aa77f72cc Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Wed, 30 Aug 2023 15:18:57 +0300 Subject: [PATCH 4/6] Generate cluster config dir and support files for aws clusters too --- deployer/generate/generate_aws_cluster.py | 63 ++++++++++++++--------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/deployer/generate/generate_aws_cluster.py b/deployer/generate/generate_aws_cluster.py index 14e8f8165..65c428c48 100644 --- a/deployer/generate/generate_aws_cluster.py +++ b/deployer/generate/generate_aws_cluster.py @@ -1,24 +1,24 @@ +""" +Generate required files for an AWS cluster + +Generates: +- an eksctl jsonnet file +- a .tfvars file +- An ssh-key (the private part is encrypted) +""" import os import subprocess -from pathlib import Path import jinja2 import typer from ..cli_app import app +from ..utils import print_colour +from .common import REPO_ROOT, generate_config_directory, generate_support_files -REPO_ROOT = Path(__file__).parent.parent.parent - - -def aws(cluster_name, hub_type, cluster_region): - """ - Generate required files for an AWS cluster - Generates: - - an eksctl jsonnet file - - a .tfvars file - - An ssh-key (the private part is encrypted) - """ +def generate_infra_files(vars): + cluster_name = vars["cluster_name"] with open(REPO_ROOT / "eksctl/template.jsonnet") as f: # jsonnet files have `}}` in there, which causes jinja2 to # freak out. So we use different delimiters. @@ -31,22 +31,20 @@ def aws(cluster_name, hub_type, cluster_region): variable_end_string=">>", ) + print_colour("Generating the eksctl jsonnet file...", "yellow") + jsonnet_file_path = REPO_ROOT / "eksctl" / f"{cluster_name}.jsonnet" + with open(jsonnet_file_path, "w") as f: + f.write(jsonnet_template.render(**vars)) + print_colour(f"{jsonnet_file_path} created") + + print_colour("Generating the terraform infrastructure file...", "yellow") with open(REPO_ROOT / "terraform/aws/projects/template.tfvars") as f: tfvars_template = jinja2.Template(f.read()) - vars = { - "cluster_name": cluster_name, - "hub_type": hub_type, - "cluster_region": cluster_region, - } - - with open(REPO_ROOT / "eksctl" / f"{cluster_name}.jsonnet", "w") as f: - f.write(jsonnet_template.render(**vars)) - - with open( - REPO_ROOT / "terraform/aws/projects" / f"{cluster_name}.tfvars", "w" - ) as f: + tfvars_file_path = REPO_ROOT / "terraform/aws/projects" / f"{cluster_name}.tfvars" + with open(tfvars_file_path, "w") as f: f.write(tfvars_template.render(**vars)) + print_colour(f"{tfvars_file_path} created") subprocess.check_call( [ @@ -89,4 +87,19 @@ def generate_aws_cluster( """ Automatically generate the files required to setup a new cluster on AWS """ - aws(cluster_name, hub_type, cluster_region) + + # These are the variables needed by the templates used to generate the cluster config file + # and support files + vars = { + "cluster_name": cluster_name, + "hub_type": hub_type, + "cluster_region": cluster_region, + } + + generate_infra_files(vars) + + # Automatically generate the config directory + cluster_config_directory = generate_config_directory(vars) + + # Generate the support files + generate_support_files(cluster_config_directory, vars) From 45cdccb1552b634ad2ffd674472e7c3de6121e09 Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Wed, 30 Aug 2023 15:29:29 +0300 Subject: [PATCH 5/6] Rm any prior refs to gcp from the common config --- deployer/generate/common.py | 12 ++++-------- deployer/generate/generate_gcp_cluster.py | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/deployer/generate/common.py b/deployer/generate/common.py index 702b201ee..53adf4323 100644 --- a/deployer/generate/common.py +++ b/deployer/generate/common.py @@ -11,11 +11,11 @@ REPO_ROOT = Path(__file__).parent.parent.parent -def generate_cluster_config_file(cluster_config_directory, vars): +def generate_cluster_config_file(cluster_config_directory, provider, vars): """ Generates the `config//cluster.yaml` config """ - with open(REPO_ROOT / "config/clusters/templates/gcp/cluster.yaml") as f: + with open(REPO_ROOT / f"config/clusters/templates/{provider}/cluster.yaml") as f: cluster_yaml_template = jinja2.Template(f.read()) with open(cluster_config_directory / "cluster.yaml", "w") as f: f.write(cluster_yaml_template.render(**vars)) @@ -71,12 +71,8 @@ def generate_support_files(cluster_config_directory, vars): def generate_config_directory(vars): """ - Generates the required `config` directory for hubs on a GCP cluster - - Generates the following files: - - `config//cluster.yaml` - - `config//support.values.yaml` - - `config//enc-support.secret.values.yaml` + Generates the required `config` directory for hubs on a cluster if it doesn't exit + and returns its name. """ cluster_config_directory = REPO_ROOT / "config/clusters" / vars["cluster_name"] diff --git a/deployer/generate/generate_gcp_cluster.py b/deployer/generate/generate_gcp_cluster.py index 371451348..ae5136302 100644 --- a/deployer/generate/generate_gcp_cluster.py +++ b/deployer/generate/generate_gcp_cluster.py @@ -71,7 +71,7 @@ def generate_gcp_cluster( cluster_config_directory = generate_config_directory(vars) # Create the cluster config directory and initial `cluster.yaml` file - generate_cluster_config_file(cluster_config_directory, vars) + generate_cluster_config_file(cluster_config_directory, "gcp", vars) # Generate the support files generate_support_files(cluster_config_directory, vars) From 658b8b1a48a5d36171ac8e7bbb08a12912c38efb Mon Sep 17 00:00:00 2001 From: Georgiana Dolocan Date: Wed, 30 Aug 2023 16:11:08 +0300 Subject: [PATCH 6/6] Allow calling generate cluster with no options, and gather tham from the prompt --- deployer/generate/generate_gcp_cluster.py | 32 ++++++++++++++++------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/deployer/generate/generate_gcp_cluster.py b/deployer/generate/generate_gcp_cluster.py index ae5136302..4f5820848 100644 --- a/deployer/generate/generate_gcp_cluster.py +++ b/deployer/generate/generate_gcp_cluster.py @@ -11,6 +11,7 @@ """ import jinja2 import typer +from typing_extensions import Annotated from ..cli_app import app from ..utils import print_colour @@ -43,20 +44,28 @@ def generate_terraform_file(vars): @app.command() def generate_gcp_cluster( - cluster_name: str = typer.Option(..., prompt="Name of the cluster"), - cluster_region: str = typer.Option(..., prompt="Cluster region"), - project_id: str = typer.Option(..., prompt="Project ID of the GCP project"), - hub_type: str = typer.Option( - ..., prompt="Type of hub. Choose from `basehub` or `daskhub`" - ), - hub_name: str = typer.Option(..., prompt="Name of the first hub"), + cluster_name: Annotated[ + str, typer.Option(prompt="Please type the name of the new cluster") + ], + project_id: Annotated[ + str, typer.Option(prompt="Please insert the Project ID of the GCP project") + ], + hub_name: Annotated[ + str, + typer.Option( + prompt="Please insert the name of first hub to add to the cluster" + ), + ], + cluster_region: Annotated[ + str, typer.Option(prompt="Please insert the name of the cluster region") + ] = "us-central1", + hub_type: Annotated[ + str, typer.Option(prompt="Please insert the hub type of the first hub") + ] = "basehub", ): """ Automatically generates the initial files, required to setup a new cluster on GCP """ - # Automatically generate the terraform config file - generate_terraform_file(cluster_name, cluster_region, project_id, hub_type) - # These are the variables needed by the templates used to generate the cluster config file # and support files vars = { @@ -67,6 +76,9 @@ def generate_gcp_cluster( "hub_name": hub_name, } + # Automatically generate the terraform config file + generate_terraform_file(vars) + # Automatically generate the config directory cluster_config_directory = generate_config_directory(vars)