diff --git a/tdrive/backend/node/src/cli/cmds/db.ts b/tdrive/backend/node/src/cli/cmds/db.ts new file mode 100644 index 000000000..a4de665ce --- /dev/null +++ b/tdrive/backend/node/src/cli/cmds/db.ts @@ -0,0 +1,14 @@ +import { CommandModule } from "yargs"; + +const command: CommandModule = { + describe: "Manage db migrations", + command: "db ", + builder: yargs => + yargs.commandDir("db_cmds", { + visit: commandModule => commandModule.default, + }), + // eslint-disable-next-line @typescript-eslint/no-empty-function + handler: () => {}, +}; + +export default command; diff --git a/tdrive/backend/node/src/cli/cmds/db_cmds/migrate-db.ts b/tdrive/backend/node/src/cli/cmds/db_cmds/migrate-db.ts new file mode 100644 index 000000000..47d5da60d --- /dev/null +++ b/tdrive/backend/node/src/cli/cmds/db_cmds/migrate-db.ts @@ -0,0 +1,179 @@ +import yargs from "yargs"; +import runWithPlatform from "../../lib/run-with-platform"; +import runWithLoggerLevel from "../../utils/run-with-logger-level"; +import gr from "../../../services/global-resolver"; +type CLIArgs = { + name: string; +}; + +// eslint-disable-next-line @typescript-eslint/ban-types +const command: yargs.CommandModule = { + command: "migrate", + builder: { + name: { + default: "", + type: "string", + description: "Entity name to migrate", + }, + output: { + default: "", + type: "string", + description: "Migration output log", + }, + }, + describe: + "command to export everything inside a company (publicly data only available to a new member)", + // eslint-disable-next-line @typescript-eslint/no-unused-vars + handler: async argv => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + await runWithPlatform("Re-index", async ({ spinner, platform }) => { + return await runWithLoggerLevel( + argv.verboseDuringRun + ? (argv.verboseDuringRun as number) > 1 + ? "debug" + : "info" + : undefined, + async () => { + const targetEntity = argv.name; + const targetService = await gr.services["users"]; + const entityClass = targetService.repository.entityType; + const entity = entityClass.prototype; + // console.log("Migrating entity: \n", JSON.stringify(entity), "\n"); + const schema = JSON.parse(await gr.database.getConnector().migrate(targetEntity)); + // Define a mapping between JavaScript data types and PostgreSQL data types + const dataTypeMapping = { + string: "text", + number: "bigint", + boolean: "boolean", + object: "jsonb", // Assuming JSON objects are stored as jsonb in PostgreSQL + date: "timestamp", // Assuming dates are stored as timestamps in PostgreSQL + array: "jsonb[]", // Assuming arrays are stored as JSON arrays in PostgreSQL + float: "real", // Assuming floating-point numbers are stored as real in PostgreSQL + double: "double precision", // Assuming double-precision floating-point numbers are stored as double precision in PostgreSQL + int: "integer", // Assuming integer numbers are stored as integer in PostgreSQL + bigint: "bigint", // Assuming big integer numbers are stored as bigint in PostgreSQL + smallint: "smallint", // Assuming small integer numbers are stored as smallint in PostgreSQL + decimal: "numeric", // Assuming decimal numbers are stored as numeric in PostgreSQL + encoded_json: "text", // Assuming encoded JSON objects are stored as text in PostgreSQL + }; + console.log("\n ##################################### \n"); + + // Compare schema with entity + const addedColumns = []; + const deletedColumns = []; + const changedColumns = []; + + // Check for added columns and renamed columns + Object.keys(entity._columns).forEach(columnName => { + const columnExistsInSchema = schema.some( + schemaField => schemaField.column_name === columnName, + ); + if (!columnExistsInSchema) { + addedColumns.push(columnName); + } else { + const schemaField = schema.find(field => field.column_name === columnName); + const entityField = entity._columns[columnName]; + + // Check if the column has a 'rename' option + const renameOption = entityField.options && entityField.options.rename; + const renamedColumnName = renameOption ? renameOption : columnName; + const columnAlreadyRenamed = schema.some( + schemaField => schemaField.column_name === renamedColumnName, + ); + if (columnAlreadyRenamed && renameOption) { + console.log( + `⚠️ Column '${columnName}' renamed to '${renamedColumnName}, please update the entity and remove the 'rename' option from the column definition: ${columnAlreadyRenamed}, ${renameOption}`, + ); + } + + // Map JavaScript data types to PostgreSQL data types if needed + const mappedEntityType = dataTypeMapping[entityField.type] || entityField.type; + + if ( + (schemaField.column_name !== renamedColumnName && !columnAlreadyRenamed) || + schemaField.data_type !== mappedEntityType + ) { + changedColumns.push({ + columnName, + renamedColumnName, + schemaType: schemaField.data_type, + entityType: mappedEntityType, + }); + } + } + }); + + // Check for deleted columns (including renamed columns) + schema.forEach(schemaField => { + const columnName = schemaField.column_name; + const columnExistsInEntity = entity._columns.hasOwnProperty(columnName); + if (!columnExistsInEntity) { + // Check if the column was renamed in the entity options + const renamedColumn = Object.values(entity._columns).find( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (field: any) => field?.options && field.options.rename === columnName, + ); + if (!renamedColumn) { + deletedColumns.push(columnName); + } + } + }); + + // Output changes + console.log("\nChanges detected:\n"); + + if (addedColumns.length > 0) { + console.log("Added columns:"); + addedColumns.forEach(columnName => console.log(`- ${columnName}`)); + } else { + console.log("No new columns added"); + } + + if (deletedColumns.length > 0) { + console.log("\nDeleted columns:"); + deletedColumns.forEach(columnName => console.log(`- ${columnName}`)); + } else { + console.log("No columns deleted"); + } + + if (changedColumns.length > 0) { + console.log("\nChanged columns:"); + changedColumns.forEach(({ columnName, renamedColumnName, schemaType, entityType }) => { + if (columnName !== renamedColumnName) { + console.log(`- Column '${columnName}' renamed to '${renamedColumnName}'`); + } + if (schemaType !== entityType) { + console.log( + `- Column '${columnName}' type changed from '${schemaType}' to '${entityType}'`, + ); + } + }); + } else { + console.log("No columns changed"); + } + + // Show appropriate PostgreSQL queries + console.log("\nPostgreSQL Queries:"); + addedColumns.forEach(columnName => + console.log( + `ALTER TABLE public."${targetEntity}" ADD COLUMN ${columnName} ;`, + ), + ); + deletedColumns.forEach(columnName => + console.log(`ALTER TABLE public."${targetEntity}" DROP COLUMN ${columnName};`), + ); + changedColumns.forEach(({ columnName, renamedColumnName }) => { + if (columnName !== renamedColumnName) { + console.log( + `ALTER TABLE public."${targetEntity}" RENAME COLUMN ${columnName} TO ${renamedColumnName};`, + ); + } + }); + console.log("\n"); + }, + ); + }); + }, +}; + +export default command; diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts index ff6e05c32..0a2bdb7af 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts @@ -13,6 +13,8 @@ export abstract class AbstractConnector implements abstract drop(): Promise; + abstract migrate(name: string): Promise; + abstract createTable( entity: EntityDefinition, columns: { [name: string]: ColumnDefinition }, diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/cassandra.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/cassandra.ts index 956f41127..a5736bdfc 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/cassandra.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/cassandra.ts @@ -74,6 +74,10 @@ export class CassandraConnector extends AbstractConnector { + return `checking ${name}: nothing changed...`; + } + createKeyspace(): Promise { const query = `CREATE KEYSPACE IF NOT EXISTS ${this.options.keyspace} WITH replication = {'class': 'NetworkTopologyStrategy', 'datacenter1': '2'} AND durable_writes = true;`; logger.info(query); diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts index 4be73d878..86e65d657 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts @@ -27,6 +27,12 @@ export interface Connector extends Initializable { */ disconnect(): Promise; + /** + * Migrate the database + */ + + migrate(name: string): Promise; + /** * Get the type of connector */ diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts index 05755448b..a1b24b377 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts @@ -25,6 +25,10 @@ export class MongoConnector extends AbstractConnector { return this; } + async migrate(name: string): Promise { + return `checking ${name}: nothing changed...`; + } + async connect(): Promise { if (this.client) { return this; diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts index 94dcd6a43..921c6099e 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts @@ -56,6 +56,15 @@ export class PostgresConnector extends AbstractConnector { + const query = `SELECT column_name, data_type, character_maximum_length, is_nullable + FROM information_schema.columns + WHERE table_name = '${name}'; + `; + const tableSchema = await this.client.query(query); + return `${JSON.stringify(tableSchema.rows)}`; + } + async init(): Promise { if (!this.client) { await this.connect(); diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/types.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/types.ts index dbf14401b..c7401c31d 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/types.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/types.ts @@ -25,6 +25,7 @@ export type ColumnOptions = { order?: "ASC" | "DESC"; generator?: ColumnType; onUpsert?: (value: any) => any; + rename?: string; }; export type ColumnType =