deployment to ansible

Bug: T342799

.gitattributes

@@ -1,2 +1,3 @@
-secrets.sh filter=git-crypt diff=git-crypt
+ansible/vars/eqiad1-secrets.yaml filter=git-crypt diff=git-crypt
+ansible/files/csi-secret-cinderplugin.yaml.eqiad1.crypt filter=git-crypt diff=git-crypt
 terraform/secrets.tf filter=git-crypt diff=git-crypt

.gitignore

@@ -5,3 +5,4 @@ dbs.yaml
 terraform/kube.config
 terraform.tfstate
 terraform.tfstate.backup
+.env/

README.md

@@ -1,54 +1,22 @@
-# if infrastructure isn't setup, start here
-## from an openstack control node:
+### Deploy
+## from the superset bastion node:
 cd terraform
 terraform init
-terraform apply -var datacenter="eqiad1"
-
-## from local:
-`git clone https://github.com/kubernetes/cloud-provider-openstack.git`
-
-cloud.conf:
-```
-[Global]
-application-credential-id = ${APPLICATION_CRED_ID}
-application-credential-secret = ${APPLICATION_CRED_SECRET}
-domain-name = default
-auth-url = https://openstack.eqiad1.wikimediacloud.org:25000/v3
-tenant-id = superset
-region = eqiad1-r
-```
-
-```
-cd cloud-provider-openstack
-base64 -w 0 ../cloud.conf ; echo
-vim manifests/cinder-csi-plugin/csi-secret-cinderplugin.yaml # replace cloud.conf 64 with above
-kubectl create -f manifests/cinder-csi-plugin/csi-secret-cinderplugin.yaml
-kubectl -f manifests/cinder-csi-plugin/ apply
-```
-
-sc.yaml:
-```
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: standard
-  annotations:
-    storageclass.kubernetes.io/is-default-class: "true"
-provisioner: cinder.csi.openstack.org
-parameters:
-  availability: nova
-```
-
-`kubectl apply -f sc.yaml`
+terraform apply -var datacenter="<codfw1dev|eqiad1>"
 
+if a new database was created update currentDb and oldDB values in ansible/vars/eqiad1.yaml
 
 # When k8s is setup, start here
-To install run `deploy.sh install`
+To install run `deploy.sh <codfw1dev|eqiad1> [migrate]`
 
-Create OAuth role:
+## Disaster recovery deploy
+after deploy.sh Create OAuth role:
 all query access on all_query_access
 
-# To backup and restore the db:
+# To migrate the db:
+`deploy.sh <codfw1dev|eqiad1> migrate`
+
+# manual db backup and restore:
 in Horizon create a new trove database:
 Volume Size: 8
 Datastore: mysql 5.7.29

ansible/db-migrate.yaml

@@ -0,0 +1,42 @@
+---
+
+- name: Migrate old db to new
+  gather_facts: false
+  hosts: localhost
+  tasks:
+    - name: include env vars
+      include_vars: "vars/{{ datacenter }}.yaml"
+
+    - name: include secret env vars
+      include_vars: "vars/{{ datacenter }}-secrets.yaml"
+
+    - name: Temporary directory for mysqldump
+      ansible.builtin.tempfile:
+        state: directory
+        suffix: mysqldump
+      register: dump_dir
+      changed_when: False
+
+    - name: Dump old database
+      community.mysql.mysql_db:
+        login_host: "{{ oldDb }}"
+        login_user: superset
+        login_password: "{{ dbPass }}"
+        state: dump
+        name: all
+        target: "{{ dump_dir.path }}/dump.sql"
+
+    - name: Restore to new database
+      community.mysql.mysql_db:
+        login_host: "{{ currentDb }}"
+        login_user: superset
+        login_password: "{{ dbPass }}"
+        name: all
+        state: import
+        target: "{{ dump_dir.path }}/dump.sql"
+
+    - name: Delete temporary directory
+      ansible.builtin.file:
+        state: absent
+        path: "{{ dump_dir.path }}"
+      changed_when: False

ansible/files/csi-secret-cinderplugin.yaml.codfw1dev.crypt

@@ -0,0 +1,10 @@
+# This YAML file contains secret objects,
+# which are necessary to run csi cinder plugin.
+
+kind: Secret
+apiVersion: v1
+metadata:
+  name: cloud-config
+  namespace: kube-system
+data:
+  cloud.conf: W0dsb2JhbF0KYXBwbGljYXRpb24tY3JlZGVudGlhbC1pZCA9IGQ3NWI0MmE0NWVhYTQ4ZWE5MjhhMzE2NDg0ZTI1NGI0CmFwcGxpY2F0aW9uLWNyZWRlbnRpYWwtc2VjcmV0ID0gdHNDeGZfclV1RmVWY2tIOV9OMjEwRF9fZDZUZWFoSEdwWVQ0NEIwT0MyT29xdEdxV292QktuclRta3U2aDZqVjN4eF81ZUdndkV6UEtQSkpyM3dvRncKZG9tYWluLW5hbWUgPSBkZWZhdWx0CmF1dGgtdXJsID0gaHR0cHM6Ly9vcGVuc3RhY2suY29kZncxZGV2Lndpa2ltZWRpYWNsb3VkLm9yZzoyNTAwMC92Mwp0ZW5hbnQtaWQgPSBrOHMtZGV2CnJlZ2lvbiA9IGNvZGZ3MWRldi1yCg==

ansible/files/sc.yaml

@@ -0,0 +1,11 @@
+---
+
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: standard
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: cinder.csi.openstack.org
+parameters:
+  availability: nova

ansible/superset-deploy.yaml

@@ -1,21 +1,15 @@
 ---
 
-- name: Deploy superset to k8s cluster
-  gather_facts: false
-  hosts: localhost
-  tasks:
-    - name: Touch a file
-      ansible.builtin.file:
-        path: /root/touchfile
-        state: touch
-
 - name: Deploy superset to k8s cluster
   gather_facts: false
   hosts: localhost
   tasks:
     - name: include env vars
       include_vars: vars/{{ datacenter }}.yaml
 
+    - name: include secret env vars
+      include_vars: vars/{{ datacenter }}-secrets.yaml
+
     - name: Temporary directory for cloud provider
       ansible.builtin.tempfile:
         state: directory
@@ -58,3 +52,18 @@
       kubernetes.core.k8s:
         state: present
         src: "files/sc.yaml"
+
+#    - name: Add superset chart repo
+#      kubernetes.core.helm_repository:
+#        name: superset
+#        repo_url: "https://apache.github.io/superset"
+
+    - name: Deploy superset
+      kubernetes.core.helm:
+        name: superset
+        chart_repo_url: "https://apache.github.io/superset"
+        chart_ref: superset
+        chart_version: 0.10.0
+        release_namespace: default
+        values: "{{ lookup('template', 'templates/values.yaml.j2') | from_yaml }}"
+        #dependency_update: true

values.yaml-template → ansible/templates/values.yaml.j2

@@ -27,8 +27,8 @@ configOverrides:
             'token_key':'access_token',
             'remote_app': {
                 # auth only callback mediawiki gives email and realname
-                'client_id':'0c0ca9df92d6da2683a84dc2efc72640',
-                'client_secret':'${OAUTH_SECRET}',
+                'client_id':'{{ oauthClient }}',
+                'client_secret':'{{ oauthSecret }}',
                 'client_kwargs':{
                     'scope': 'mwoauth-authonlyprivate'
                 },
@@ -75,4 +75,32 @@ configOverrides:
     # Authorized User
     AUTH_USER_REGISTRATION = True
 
-    SQLALCHEMY_DATABASE_URI = 'mysql://superset:${DB_PASS}@2xopzht2475.svc.trove.eqiad1.wikimedia.cloud/superset'
+    SQLALCHEMY_DATABASE_URI = 'mysql://superset:{{ dbPass }}@{{ currentDb }}/superset'
+
+extraConfigs:
+  import_datasources.yaml: |
+    databases:
+    - database_name: s1
+      password: {{ replicaPass }}
+      sqlalchemy_uri: mysql+mysqldb://[email protected]:3306/information_schema?ssl=1
+    - database_name: s2
+      password: {{ replicaPass }}
+      sqlalchemy_uri: mysql+mysqldb://[email protected]:3306/information_schema?ssl=1
+    - database_name: s3
+      password: {{ replicaPass }}
+      sqlalchemy_uri: mysql+mysqldb://[email protected]:3306/information_schema?ssl=1
+    - database_name: s4
+      password: {{ replicaPass }}
+      sqlalchemy_uri: mysql+mysqldb://[email protected]:3306/information_schema?ssl=1
+    - database_name: s5
+      password: {{ replicaPass }}
+      sqlalchemy_uri: mysql+mysqldb://[email protected]:3306/information_schema?ssl=1
+    - database_name: s6
+      password: {{ replicaPass }}
+      sqlalchemy_uri: mysql+mysqldb://[email protected]:3306/information_schema?ssl=1
+    - database_name: s7
+      password: {{ replicaPass }}
+      sqlalchemy_uri: mysql+mysqldb://[email protected]:3306/information_schema?ssl=1
+    - database_name: s8
+      password: {{ replicaPass }}
+      sqlalchemy_uri: mysql+mysqldb://[email protected]:3306/information_schema?ssl=1

ansible/vars/codfw1dev-secrets.yaml

@@ -0,0 +1,6 @@
+---
+
+replicaPass: 'OwWPd58ehbisJL4y'
+oauthClient: 'a008221205f74d2f50f6da00511ec287'
+oauthSecret: '1ee7bcabdfe01efecead659bc8cc4aa393c7edbe'
+dbPass: 'jfq3qu80ua38ru8ua3w0'

ansible/vars/codfw1dev.yaml

@@ -0,0 +1,4 @@
+---
+
+currentDb: kdmnanixf6n.svc.trove.codfw1dev.wikimedia.cloud 
+oldDb: kzh57llbocj.svc.trove.codfw1dev.wikimedia.cloud

ansible/vars/eqiad1.yaml

@@ -0,0 +1,4 @@
+---
+
+currentDb: 3paubms4bt4.svc.trove.eqiad1.wikimedia.cloud
+oldDb: 2xopzht2475.svc.trove.eqiad1.wikimedia.cloud

deploy.sh

@@ -1,27 +1,54 @@
 #!/bin/bash
 
 set -e
-datacenter='eqiad1'
+
+migrate='false'
+
+if [ "${1}" = 'eqiad1' ]
+then
+  datacenter=${1}
+elif [ "${1}" = 'codfw1dev' ]
+then
+  datacenter=${1}
+else
+  echo "Please enter datacenter."
+  echo "Usage:"
+  echo "${0} <eqiad1|codfw1dev>"
+  exit
+fi
+
+if [ "${2}" = 'migrate' ]
+then
+  migrate='true'
+fi
 
 
 if ! command -v kubectl ; then
   echo "please install kubectl"
   exit 1
 fi
 
-if ! command -v terraform ; then
-  echo "please install terraform"
+if ! command -v helm ; then
+  echo "please install helm"
+  exit 1
+fi
+
+if ! command -v mysqldump ; then
+  echo "please install mariadb-client"
   exit 1
 fi
 
 python3 -m venv .venv/deploy
 source .venv/deploy/bin/activate
-pip install ansible==8.1.0 kubernetes==26.1.0
+pip install ansible==8.1.0 kubernetes==26.1.0 PyMySQL==1.1.0
 
-cd terraform
-terraform init
-terraform apply -var datacenter=${datacenter}  # -auto-approve
-export KUBECONFIG=$(pwd)/kube.config
+export KUBECONFIG=$(pwd)/terraform/kube.config
 
-cd ../ansible
+cd ansible
 ansible-playbook superset-deploy.yaml --extra-vars "datacenter=${datacenter}"
+
+if [ "${migrate}" = 'true' ]
+then
+  echo "migrating!"
+  ansible-playbook db-migrate.yaml --extra-vars "datacenter=${datacenter}"
+fi

terraform/db.tf

@@ -1,11 +1,11 @@
 resource "openstack_db_instance_v1" "superset" {
   region    = var.region[var.datacenter]
-  name      = "superset-tf"
-  flavor_id = "bb8bee7e-d8f9-460b-8344-74f745c139b9"
-  size      = 4
+  name      = "superset${var.name[var.datacenter]}"
+  flavor_id = var.db_flavor_uuid[var.datacenter]
+  size      = var.db_size[var.datacenter]
 
   network {
-    uuid = "7425e328-560c-4f00-8e99-706f3fb90bb4"
+    uuid = var.network_uuid[var.datacenter]
   }
 
   user {

terraform/vars.tf

@@ -22,14 +22,14 @@ variable "auth-url" {
 variable "tenant_id" {
   type = map(any)
   default = {
-    "codfw1dev" = ""
+    "codfw1dev" = "k8s-dev"
     "eqiad1"    = "superset"
   }
 }
 variable "application_credential_id" {
   type = map(any)
   default = {
-    "codfw1dev" = ""
+    "codfw1dev" = "d75b42a45eaa48ea928a316484e254b4"
     "eqiad1"    = "91fc0e330c734a439be74d335627dfed"
   }
 }