Merge pull request #3061 from GeorgianaElena/templates

[deployer] Have the aws cluster generator also create cluster config directory and support files

.pre-commit-config.yaml

@@ -53,7 +53,7 @@ repos:
     hooks:
       - id: sops-encryption
         # Add files here if they contain the word 'secret' but should not be encrypted
-        exclude: secrets\.md|helm-charts/support/templates/prometheus-ingres-auth/secret\.yaml|helm-charts/basehub/templates/dex/secret\.yaml|helm-charts/basehub/templates/static/secret\.yaml|config/clusters/templates/gcp/support\.secret\.values\.yaml|helm-charts/basehub/templates/ingress-auth/secret\.yaml
+        exclude: secrets\.md|helm-charts/support/templates/prometheus-ingres-auth/secret\.yaml|helm-charts/basehub/templates/dex/secret\.yaml|helm-charts/basehub/templates/static/secret\.yaml|config/clusters/templates/common/support\.secret\.values\.yaml|helm-charts/basehub/templates/ingress-auth/secret\.yaml
 
 # pre-commit.ci config reference: https://pre-commit.ci/#configuration
 ci:

deployer/generate/common.py

@@ -0,0 +1,91 @@
+import os
+import secrets
+import string
+import subprocess
+from pathlib import Path
+
+import jinja2
+
+from ..utils import print_colour
+
+REPO_ROOT = Path(__file__).parent.parent.parent
+
+
+def generate_cluster_config_file(cluster_config_directory, provider, vars):
+    """
+    Generates the `config/<cluster_name>/cluster.yaml` config
+    """
+    with open(REPO_ROOT / f"config/clusters/templates/{provider}/cluster.yaml") as f:
+        cluster_yaml_template = jinja2.Template(f.read())
+    with open(cluster_config_directory / "cluster.yaml", "w") as f:
+        f.write(cluster_yaml_template.render(**vars))
+
+
+def generate_support_files(cluster_config_directory, vars):
+    """
+    Generates files related to support components.
+
+    They are required to deploy the support chart for the cluster
+    and to configure the Prometheus instance.
+
+    Generates:
+        - `config/<cluster_name>/support.values.yaml`
+        - `config/<cluster_name>/enc-support.secret.values.yaml`
+    """
+    # Generate the suppport values file `support.values.yaml`
+    print_colour("Generating the support values file...", "yellow")
+    with open(REPO_ROOT / "config/clusters/templates/common/support.values.yaml") as f:
+        support_values_yaml_template = jinja2.Template(f.read())
+
+    with open(cluster_config_directory / "support.values.yaml", "w") as f:
+        f.write(support_values_yaml_template.render(**vars))
+    print_colour(f"{cluster_config_directory}/support.values.yaml created")
+
+    # Generate and encrypt prometheus credentials into `enc-support.secret.values.yaml`
+    print_colour("Generating the prometheus credentials encrypted file...", "yellow")
+    alphabet = string.ascii_letters + string.digits
+    credentials = {
+        "username": "".join(secrets.choice(alphabet) for i in range(64)),
+        "password": "".join(secrets.choice(alphabet) for i in range(64)),
+    }
+    with open(
+        REPO_ROOT / "config/clusters/templates/common/support.secret.values.yaml"
+    ) as f:
+        support_secret_values_yaml_template = jinja2.Template(f.read())
+    with open(cluster_config_directory / "enc-support.secret.values.yaml", "w") as f:
+        f.write(support_secret_values_yaml_template.render(**credentials))
+
+    # Encrypt the private key
+    subprocess.check_call(
+        [
+            "sops",
+            "--in-place",
+            "--encrypt",
+            cluster_config_directory / "enc-support.secret.values.yaml",
+        ]
+    )
+    print_colour(
+        f"{cluster_config_directory}/enc-support.values.yaml created and encrypted"
+    )
+
+
+def generate_config_directory(vars):
+    """
+    Generates the required `config` directory for hubs on a cluster if it doesn't exit
+    and returns its name.
+    """
+    cluster_config_directory = REPO_ROOT / "config/clusters" / vars["cluster_name"]
+
+    print_colour(
+        f"Checking if cluster config directory {cluster_config_directory} exists...",
+        "yellow",
+    )
+    if os.path.exists(cluster_config_directory):
+        print_colour(f"{cluster_config_directory} already exists.")
+        return cluster_config_directory
+
+    # Create the cluster config directory and initial `cluster.yaml` file
+    os.makedirs(cluster_config_directory)
+    print_colour(f"{cluster_config_directory} created")
+
+    return cluster_config_directory

deployer/generate/generate_aws_cluster.py

@@ -1,24 +1,24 @@
+"""
+Generate required files for an AWS cluster
+
+Generates:
+- an eksctl jsonnet file
+- a .tfvars file
+- An ssh-key (the private part is encrypted)
+"""
 import os
 import subprocess
-from pathlib import Path
 
 import jinja2
 import typer
 
 from ..cli_app import app
+from ..utils import print_colour
+from .common import REPO_ROOT, generate_config_directory, generate_support_files
 
-REPO_ROOT = Path(__file__).parent.parent.parent
-
-
-def aws(cluster_name, hub_type, cluster_region):
-    """
-    Generate required files for an AWS cluster
 
-    Generates:
-    - an eksctl jsonnet file
-    - a .tfvars file
-    - An ssh-key (the private part is encrypted)
-    """
+def generate_infra_files(vars):
+    cluster_name = vars["cluster_name"]
     with open(REPO_ROOT / "eksctl/template.jsonnet") as f:
         # jsonnet files have `}}` in there, which causes jinja2 to
         # freak out. So we use different delimiters.
@@ -31,22 +31,20 @@ def aws(cluster_name, hub_type, cluster_region):
             variable_end_string=">>",
         )
 
+    print_colour("Generating the eksctl jsonnet file...", "yellow")
+    jsonnet_file_path = REPO_ROOT / "eksctl" / f"{cluster_name}.jsonnet"
+    with open(jsonnet_file_path, "w") as f:
+        f.write(jsonnet_template.render(**vars))
+    print_colour(f"{jsonnet_file_path} created")
+
+    print_colour("Generating the terraform infrastructure file...", "yellow")
     with open(REPO_ROOT / "terraform/aws/projects/template.tfvars") as f:
         tfvars_template = jinja2.Template(f.read())
 
-    vars = {
-        "cluster_name": cluster_name,
-        "hub_type": hub_type,
-        "cluster_region": cluster_region,
-    }
-
-    with open(REPO_ROOT / "eksctl" / f"{cluster_name}.jsonnet", "w") as f:
-        f.write(jsonnet_template.render(**vars))
-
-    with open(
-        REPO_ROOT / "terraform/aws/projects" / f"{cluster_name}.tfvars", "w"
-    ) as f:
+    tfvars_file_path = REPO_ROOT / "terraform/aws/projects" / f"{cluster_name}.tfvars"
+    with open(tfvars_file_path, "w") as f:
         f.write(tfvars_template.render(**vars))
+    print_colour(f"{tfvars_file_path} created")
 
     subprocess.check_call(
         [
@@ -89,4 +87,19 @@ def generate_aws_cluster(
     """
     Automatically generate the files required to setup a new cluster on AWS
     """
-    aws(cluster_name, hub_type, cluster_region)
+
+    # These are the variables needed by the templates used to generate the cluster config file
+    # and support files
+    vars = {
+        "cluster_name": cluster_name,
+        "hub_type": hub_type,
+        "cluster_region": cluster_region,
+    }
+
+    generate_infra_files(vars)
+
+    # Automatically generate the config directory
+    cluster_config_directory = generate_config_directory(vars)
+
+    # Generate the support files
+    generate_support_files(cluster_config_directory, vars)

deployer/generate/generate_gcp_cluster.py

@@ -1,111 +1,73 @@
-import os
-import secrets
-import string
-import subprocess
-from pathlib import Path
+"""
+Generates the ` terraform file required to create a GCP cluster
+and the required `config` directory for hubs on a GCP cluster.
 
+Generates the following files:
+- terraform/gcp/projects/<cluster_name>.tfvars`
+- `config/<cluster_name>/cluster.yaml`
+- `config/<cluster_name>/support.values.yaml`
+- `config/<cluster_name>/enc-support.secret.values.yaml`
+
+"""
 import jinja2
 import typer
+from typing_extensions import Annotated
 
 from ..cli_app import app
 from ..utils import print_colour
-
-REPO_ROOT = Path(__file__).parent.parent.parent
+from .common import (
+    REPO_ROOT,
+    generate_cluster_config_file,
+    generate_config_directory,
+    generate_support_files,
+)
 
 
-def generate_terraform_file(cluster_name, cluster_region, project_id, hub_type):
+def generate_terraform_file(vars):
     """
     Generates the `terraform/gcp/projects/<cluster_name>.tfvars` terraform file
     required to create a GCP cluster
     """
-    with open(REPO_ROOT / f"terraform/gcp/projects/{hub_type}-template.tfvars") as f:
-        tfvars_template = jinja2.Template(f.read())
-
-    vars = {
-        "cluster_name": cluster_name,
-        "cluster_region": cluster_region,
-        "project_id": project_id,
-    }
-
-    print_colour("Generating the terraform infrastructure file...", "yellow")
-    with open(
-        REPO_ROOT / "terraform/gcp/projects" / f"{cluster_name}.tfvars", "w"
-    ) as f:
-        f.write(tfvars_template.render(**vars))
-    print_colour(f"{REPO_ROOT}/terraform/gcp/projects/{cluster_name}.tfvars created")
-
-
-def generate_cluster_config_file(cluster_config_directory, vars):
-    """
-    Generates the `config/<cluster_name>/cluster.yaml` config
-    """
-    with open(REPO_ROOT / "config/clusters/templates/gcp/cluster.yaml") as f:
-        cluster_yaml_template = jinja2.Template(f.read())
-    with open(cluster_config_directory / "cluster.yaml", "w") as f:
-        f.write(cluster_yaml_template.render(**vars))
-
-
-def generate_support_files(cluster_config_directory, vars):
-    """
-    Generates files related to support components.
-
-    They are required to deploy the support chart for the cluster
-    and to configure the Prometheus instance.
-
-    Generates:
-        - `config/<cluster_name>/support.values.yaml`
-        - `config/<cluster_name>/enc-support.secret.values.yaml`
-    """
-    # Generate the suppport values file `support.values.yaml`
-    print_colour("Generating the support values file...", "yellow")
-    with open(REPO_ROOT / "config/clusters/templates/gcp/support.values.yaml") as f:
-        support_values_yaml_template = jinja2.Template(f.read())
-
-    with open(cluster_config_directory / "support.values.yaml", "w") as f:
-        f.write(support_values_yaml_template.render(**vars))
-    print_colour(f"{cluster_config_directory}/support.values.yaml created")
-
-    # Generate and encrypt prometheus credentials into `enc-support.secret.values.yaml`
-    print_colour("Generating the prometheus credentials encrypted file...", "yellow")
-    alphabet = string.ascii_letters + string.digits + string.punctuation
-    credentials = {
-        "username": "".join(secrets.choice(alphabet) for i in range(64)),
-        "password": "".join(secrets.choice(alphabet) for i in range(64)),
-    }
     with open(
-        REPO_ROOT / "config/clusters/templates/gcp/support.secret.values.yaml"
+        REPO_ROOT / f'terraform/gcp/projects/{vars["hub_type"]}-template.tfvars'
     ) as f:
-        support_secret_values_yaml_template = jinja2.Template(f.read())
-    with open(cluster_config_directory / "enc-support.secret.values.yaml", "w") as f:
-        f.write(support_secret_values_yaml_template.render(**credentials))
+        tfvars_template = jinja2.Template(f.read())
 
-    # Encrypt the private key
-    subprocess.check_call(
-        [
-            "sops",
-            "--in-place",
-            "--encrypt",
-            cluster_config_directory / "enc-support.secret.values.yaml",
-        ]
-    )
-    print_colour(
-        f"{cluster_config_directory}/enc-support.values.yaml created and encrypted"
+    print_colour("Generating the terraform infrastructure file...", "yellow")
+    tfvars_file_path = (
+        REPO_ROOT / "terraform/gcp/projects" / f'{vars["cluster_name"]}.tfvars'
     )
+    with open(tfvars_file_path, "w") as f:
+        f.write(tfvars_template.render(**vars))
+    print_colour(f"{tfvars_file_path} created")
 
 
-def generate_config_directory(
-    cluster_name, cluster_region, project_id, hub_type, hub_name
+@app.command()
+def generate_gcp_cluster(
+    cluster_name: Annotated[
+        str, typer.Option(prompt="Please type the name of the new cluster")
+    ],
+    project_id: Annotated[
+        str, typer.Option(prompt="Please insert the Project ID of the GCP project")
+    ],
+    hub_name: Annotated[
+        str,
+        typer.Option(
+            prompt="Please insert the name of first hub to add to the cluster"
+        ),
+    ],
+    cluster_region: Annotated[
+        str, typer.Option(prompt="Please insert the name of the cluster region")
+    ] = "us-central1",
+    hub_type: Annotated[
+        str, typer.Option(prompt="Please insert the hub type of the first hub")
+    ] = "basehub",
 ):
     """
-    Generates the required `config` directory for hubs on a GCP cluster
-
-    Generates the following files:
-    - `config/<cluster_name>/cluster.yaml`
-    - `config/<cluster_name>/support.values.yaml`
-    - `config/<cluster_name>/enc-support.secret.values.yaml`
+    Automatically generates the initial files, required to setup a new cluster on GCP
     """
-    cluster_config_directory = REPO_ROOT / "config/clusters" / cluster_name
-
+    # These are the variables needed by the templates used to generate the cluster config file
+    # and support files
     vars = {
         "cluster_name": cluster_name,
         "hub_type": hub_type,
@@ -114,40 +76,14 @@ def generate_config_directory(
         "hub_name": hub_name,
     }
 
-    print_colour(
-        "Checking if cluster config directory {cluster_config_directory} exists...",
-        "yellow",
-    )
-    if os.path.exists(cluster_config_directory):
-        print_colour(f"{cluster_config_directory} already exists.")
-        return
+    # Automatically generate the terraform config file
+    generate_terraform_file(vars)
+
+    # Automatically generate the config directory
+    cluster_config_directory = generate_config_directory(vars)
 
     # Create the cluster config directory and initial `cluster.yaml` file
-    os.makedirs(cluster_config_directory)
-    print_colour(f"{cluster_config_directory} created")
-    generate_cluster_config_file(cluster_config_directory, vars)
+    generate_cluster_config_file(cluster_config_directory, "gcp", vars)
 
     # Generate the support files
     generate_support_files(cluster_config_directory, vars)
-
-
-@app.command()
-def generate_gcp_cluster(
-    cluster_name: str = typer.Option(..., prompt="Name of the cluster"),
-    cluster_region: str = typer.Option(..., prompt="Cluster region"),
-    project_id: str = typer.Option(..., prompt="Project ID of the GCP project"),
-    hub_type: str = typer.Option(
-        ..., prompt="Type of hub. Choose from `basehub` or `daskhub`"
-    ),
-    hub_name: str = typer.Option(..., prompt="Name of the first hub"),
-):
-    """
-    Automatically generates the initial files, required to setup a new cluster on GCP
-    """
-    # Automatically generate the terraform config file
-    generate_terraform_file(cluster_name, cluster_region, project_id, hub_type)
-
-    # Automatically generate the config directory
-    generate_config_directory(
-        cluster_name, cluster_region, project_id, hub_type, hub_name
-    )