Modify api-db and keycloak-db to support MySQL8

.github/workflows/test-db-migrations.yml

@@ -47,7 +47,7 @@ jobs:
       -
         name: Show initial migration logs
         run: |
-          docker compose -p lagoon logs api-init
+          docker compose -p lagoon logs api-db-init
       -
         name: Initiate rollback
         run: |

docker-compose.local-dev.yaml

@@ -29,6 +29,7 @@ services:
   api:
     command: ./node_modules/.bin/tsc-watch --build --incremental --onSuccess "node -r dotenv-extended/config dist/index"
     volumes:
+      - ./services/api/database:/app/services/api/database
       - ./services/api/src:/app/services/api/src
       - ./node-packages:/app/node-packages:delegated
       - /app/node-packages/commons/dist

docker-compose.yaml

@@ -5,9 +5,14 @@ volumes:
 
 services:
   api-db:
-    image: ${IMAGE_REPO:-lagoon}/api-db:${IMAGE_REPO_API_DB_TAG:-${IMAGE_REPO_TAG:-latest}}
+    image: uselagoon/mysql-8.0:latest
     ports:
       - '3366:3306'
+    command: mysqld --sql_mode=""
+    environment:
+      - MYSQL_USER=api
+      - MYSQL_PASSWORD=api
+      - MYSQL_DATABASE=infrastructure
   webhook-handler:
     image: ${IMAGE_REPO:-lagoon}/webhook-handler:${IMAGE_REPO_WEBHOOK_HANDLER_TAG:-${IMAGE_REPO_TAG:-latest}}
     ports:
@@ -156,13 +161,19 @@ services:
     environment:
       - KEYCLOAK_FRONTEND_URL=http://localhost:8088/auth/
       - [email protected]
+      - DB_VENDOR=mysql
+      - LAGOON_DB_VENDOR=mysql
       # Uncomment for local new relic tracking
       # - NEW_RELIC_LICENSE_KEY=
       # - NEW_RELIC_APP_NAME=keycloak-local
   keycloak-db:
-    image: ${IMAGE_REPO:-lagoon}/keycloak-db:${IMAGE_REPO_KEYCLOAK_DB_TAG:-${IMAGE_REPO_TAG:-latest}}
+    image: uselagoon/mysql-8.0:latest
     ports:
       - '3336:3306'
+    environment:
+      - MYSQL_USER=keycloak
+      - MYSQL_PASSWORD=keycloak
+      - MYSQL_DATABASE=keycloak
   tests-kubernetes:
     image: ${IMAGE_REPO:-lagoon}/tests:${IMAGE_REPO_TESTS_TAG:-${IMAGE_REPO_TAG:-latest}}
     environment:

services/api/database/migrations/20220908065119_initial_db.js

@@ -9,18 +9,18 @@
             .createTable('advanced_task_definition', function (table) {
                 table.increments('id').notNullable().primary();
                 table.string('name', 300).notNullable();
-                table.specificType('description', 'text').defaultTo('').notNullable();
+                table.specificType('description', 'text').notNullable();
                 table.string('image', 2000).defaultTo('');
                 table.string('service', 100);
                 table.string('type', 100).notNullable();
                 table.integer('environment');
                 table.integer('project');
-                table.string('group_name', 2000);
+                table.string('group_name', 300);
                 table.enu('permission', ['GUEST', 'DEVELOPER', 'MAINTAINER']).defaultTo('GUEST');
-                table.specificType('command', 'text').defaultTo('');
+                table.specificType('command', 'text');
                 table.string('confirmation_text', 2000);
                 table.timestamp('created').notNullable().defaultTo(knex.fn.now());
-                table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00');
+                table.timestamp('deleted').notNullable();
                 table.unique(['name', 'environment', 'project', 'group_name'], {indexName: 'name', storageEngineIndexType: 'hash'});
             })
             .createTable('advanced_task_definition_argument', function (table) {
@@ -76,8 +76,8 @@
                 table.string('name', 100);
                 table.integer('project');
                 table.enu('deploy_type', ['branch', 'pullrequest', 'promote']).notNullable();
-                table.string('deploy_base_ref', 100);
-                table.string('deploy_head_ref', 100);
+                table.string('deploy_base_ref', 250);
+                table.string('deploy_head_ref', 250);
                 table.string('deploy_title', 300);
                 table.enu('environment_type', ['production', 'development']).notNullable();
                 table.integer('auto_idle', 1).notNullable().defaultTo(1);
@@ -89,7 +89,7 @@
                 table.string('openshift_project_pattern', 300);
                 table.timestamp('updated').notNullable().defaultTo(knex.fn.now());
                 table.timestamp('created').notNullable().defaultTo(knex.fn.now());
-                table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00');
+                table.timestamp('deleted').notNullable();
                 table.unique(['project', 'name', 'deleted'], {indexName: 'project_name_deleted'});
             })
             .createTable('environment_backup', function (table) {
@@ -98,7 +98,7 @@
                 table.string('source', 300);
                 table.string('backup_id', 300).unique({indexName:'backup_id'});
                 table.timestamp('created').notNullable().defaultTo(knex.raw('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'));
-                table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00');
+                table.timestamp('deleted').notNullable();
                 table.index('environment', 'backup_environment');
             })
             .createTable('environment_fact', function (table) {
@@ -109,9 +109,9 @@
                 table.string('value', 300).notNullable();
                 table.enu('type', ['TEXT', 'URL', 'SEMVER']).defaultTo('TEXT');
                 table.string('source', 300).defaultTo('');
-                table.specificType('description', 'text').defaultTo('');
+                table.specificType('description', 'text');
                 table.timestamp('created').notNullable().defaultTo(knex.fn.now());
-                table.specificType('category', 'text').defaultTo('');
+                table.specificType('category', 'text');
                 table.boolean('key_fact').notNullable().defaultTo(0);
                 table.unique(['environment', 'name', 'source'], {indexName: 'environment_fact'});
             })
@@ -126,17 +126,17 @@
                 table.integer('environment');
                 table.string('severity', 300).defaultTo('');
                 table.decimal('severity_score', 1, 1).defaultTo(0);
-                table.string('identifier', 300).notNullable();
-                table.string('lagoon_service', 300).defaultTo('');
+                table.string('identifier', 100).notNullable();
+                table.string('lagoon_service', 100).defaultTo('');
                 table.string('source', 300).defaultTo('');
                 table.string('associated_package', 300).defaultTo('');
-                table.specificType('description', 'text').defaultTo('');
-                table.string('version', 300).defaultTo('');
+                table.specificType('description', 'text');
+                table.string('version', 100).defaultTo('');
                 table.string('fixed_version', 300).defaultTo('');
                 table.string('links', 300).defaultTo('');
                 table.json('data');
                 table.timestamp('created').notNullable().defaultTo(knex.fn.now());
-                table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00');
+                table.timestamp('deleted').notNullable();
                 table.unique(['environment', 'lagoon_service', 'version', 'identifier', 'deleted'], {indexName: 'environment'});
             })
             .createTable('environment_service', function (table) {
@@ -235,7 +235,7 @@
                 table.integer('development_environments_limit');
                 table.timestamp('created').notNullable().defaultTo(knex.fn.now());
                 table.string('private_key', 5000);
-                table.json('metadata').defaultTo('{}');
+                table.json('metadata');
                 table.string('router_pattern', 300).defaultTo(null);
             })
             .createTable('project_notification', function (table) {
@@ -251,7 +251,7 @@
                 table.string('filename', 100).notNullable();
                 table.text('s3_key').notNullable();
                 table.datetime('created').notNullable().defaultTo(knex.fn.now());
-                table.datetime('deleted').notNullable().defaultTo('0000-00-00 00:00:00');
+                table.datetime('deleted').notNullable();
             })
             .createTable('ssh_key', function (table) {
                table.increments('id').notNullable().primary();
@@ -295,7 +295,7 @@
                 table.integer('project', 11).notNullable();
                 table.integer('advanced_task_definition', 11).notNullable();
                 table.timestamp('created').notNullable().defaultTo(knex.fn.now());
-                table.timestamp('deleted').notNullable().defaultTo('0000-00-00 00:00:00');
+                table.timestamp('deleted').notNullable();
             })
         }
         else {

services/api/database/migrations/20230801000000_organizations.js

@@ -10,7 +10,7 @@ exports.up = async function(knex) {
             table.increments('id').notNullable().primary();
             table.string('name', 300).unique({indexName: 'name'});
             table.string('friendly_name', 300).notNullable();
-            table.specificType('description', 'text').defaultTo('').notNullable();
+            table.specificType('description', 'text').notNullable();
             table.integer('quota_project').defaultTo(1).notNullable();
             table.integer('quota_group').defaultTo(10).notNullable();
             table.integer('quota_notification').defaultTo(10).notNullable();

services/api/database/migrations/20241231000000_mysql8_compatibility.js

@@ -0,0 +1,57 @@
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.up = async function(knex) {
+    return knex.schema
+    .alterTable('advanced_task_definition', function (table) {
+        table.specificType('description', 'text').notNullable().alter();
+        table.string('group_name', 300).alter();
+        table.specificType('command', 'text').alter();
+    })
+    .alterTable('environment_fact', function (table) {
+        table.specificType('description', 'text').alter();
+        table.specificType('category', 'text').alter();
+    })
+    .alterTable('environment_problem', function (table) {
+        table.string('identifier', 100).notNullable().alter();
+        table.string('lagoon_service', 100).defaultTo('').alter();
+        table.specificType('description', 'text').alter();
+        table.string('version', 100).defaultTo('').alter();
+    })
+    .alterTable('organization', function (table) {
+        table.specificType('description', 'text').notNullable().alter();
+    })
+    .alterTable('project', function (table) {
+        table.json('metadata').alter();
+    })
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = async function(knex) {
+    // caveats around this are that the rollback can only work while data is still saved in keycloak attributes
+    // once we remove that duplication of attribute into keycloak, this rollback would result in data loss for group>project associations
+    // for any group project associations made after the attribute removal
+    return knex.schema
+    .alterTable('advanced_task_definition', function (table) {
+        table.specificType('description', 'text').defaultTo('').notNullable().alter();
+        table.string('group_name', 300).alter();
+        table.specificType('command', 'text').defaultTo('').alter();
+    })
+    .alterTable('environment_fact', function (table) {
+        table.specificType('description', 'text').defaultTo('').alter();
+        table.specificType('category', 'text').defaultTo('').alter();
+    })
+    .alterTable('environment_problem', function (table) {
+        table.string('identifier', 100).notNullable().alter();
+        table.string('lagoon_service', 100).defaultTo('').alter();
+        table.specificType('description', 'text').defaultTo('').alter();
+        table.string('version', 100).defaultTo('').alter();
+    })
+    .alterTable('organization', function (table) {
+        table.specificType('description', 'text').notNullable().defaultTo('').alter();
+    })
+};

services/keycloak/entrypoints/default-keycloak-entrypoint.sh

@@ -2,7 +2,7 @@
 export KC_DB=$DB_VENDOR
 export KC_DB_USERNAME=$DB_USER
 export KC_DB_PASSWORD=$DB_PASSWORD
-export KC_DB_URL=jdbc:mariadb://$DB_ADDR:3306/$DB_DATABASE
+export KC_DB_URL=jdbc:$DB_VENDOR://$DB_ADDR:3306/$DB_DATABASE
 
 export KC_PROXY=${KC_PROXY:-edge}
 export KC_HOSTNAME_STRICT=${KC_HOSTNAME_STRICT:-false}