From 74b3e14f39951f88c60c0c1478ce1362a1d323ca Mon Sep 17 00:00:00 2001 From: kontrollanten <6680299+kontrollanten@users.noreply.github.com> Date: Sat, 16 Apr 2022 16:42:57 +0200 Subject: [PATCH] server(plugins): upate without downtime Create a new folder each time a new plugin/theme is installed or updated. The folder name is created based on the package.json content hash. closes #4828 --- .../src/server/plugins-command.ts | 2 +- packages/tests/src/api/server/plugins.ts | 2 +- server/core/helpers/core-utils.ts | 16 +++++- server/core/helpers/decache.ts | 15 +----- server/core/lib/plugins/plugin-manager.ts | 48 ++++++++++++------ server/core/lib/plugins/yarn.ts | 49 ++++++++++++++++--- server/server.ts | 2 +- 7 files changed, 94 insertions(+), 40 deletions(-) diff --git a/packages/server-commands/src/server/plugins-command.ts b/packages/server-commands/src/server/plugins-command.ts index f85ef0330d26..03ff7876e512 100644 --- a/packages/server-commands/src/server/plugins-command.ts +++ b/packages/server-commands/src/server/plugins-command.ts @@ -253,6 +253,6 @@ export class PluginsCommand extends AbstractCommand { } private getPackageJSONPath (npmName: string) { - return this.server.servers.buildDirectory(join('plugins', 'node_modules', npmName, 'package.json')) + return this.server.servers.buildDirectory(join('plugins', 'latest', 'node_modules', npmName, 'package.json')) } } diff --git a/packages/tests/src/api/server/plugins.ts b/packages/tests/src/api/server/plugins.ts index f6905fe7efe0..1df22f15a817 100644 --- a/packages/tests/src/api/server/plugins.ts +++ b/packages/tests/src/api/server/plugins.ts @@ -370,7 +370,7 @@ describe('Test plugins', function () { const query = `UPDATE "application" SET "nodeABIVersion" = 1` await sqlCommand.updateQuery(query) - const baseNativeModule = server.servers.buildDirectory(join('plugins', 'node_modules', 'a-native-example')) + const baseNativeModule = server.servers.buildDirectory(join('plugins', 'latest', 'node_modules', 'a-native-example')) await removeNativeModule() await server.kill() diff --git a/server/core/helpers/core-utils.ts b/server/core/helpers/core-utils.ts index 9b40ca5be808..1eb56e99d8f4 100644 --- a/server/core/helpers/core-utils.ts +++ b/server/core/helpers/core-utils.ts @@ -7,7 +7,8 @@ import { promisify1, promisify2, promisify3 } from '@peertube/peertube-core-utils' import { exec, ExecOptions } from 'child_process' -import { ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions, scrypt } from 'crypto' +import { createHash, ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions, scrypt } from 'crypto' +import { createReadStream } from 'fs' import truncate from 'lodash-es/truncate.js' import { pipeline } from 'stream' import { URL } from 'url' @@ -261,6 +262,17 @@ function generateED25519KeyPairPromise () { }) } +function getContentHash (filePath: string): Promise { + return new Promise((resolve, reject) => { + const hash = createHash('md5') + hash.update(new Date().toString()) + const stream = createReadStream(filePath) + stream.on('error', err => reject(err)) + stream.on('data', chunk => hash.update(chunk)) + stream.on('end', () => resolve(hash.digest('hex'))) + }) +} + // --------------------------------------------------------------------------- const randomBytesPromise = promisify1(randomBytes) @@ -285,6 +297,8 @@ export { scryptPromise, + getContentHash, + randomBytesPromise, generateRSAKeyPairPromise, diff --git a/server/core/helpers/decache.ts b/server/core/helpers/decache.ts index cef3d639c182..b9ef40c2f703 100644 --- a/server/core/helpers/decache.ts +++ b/server/core/helpers/decache.ts @@ -4,18 +4,6 @@ import { Module } from 'module' import { extname } from 'path' -function decachePlugin (require: NodeRequire, libraryPath: string) { - const moduleName = find(require, libraryPath) - - if (!moduleName) return - - searchCache(require, moduleName, function (mod) { - delete require.cache[mod.id] - - removeCachedPath(mod.path) - }) -} - function decacheModule (require: NodeRequire, name: string) { const moduleName = find(require, name) @@ -31,8 +19,7 @@ function decacheModule (require: NodeRequire, name: string) { // --------------------------------------------------------------------------- export { - decacheModule, - decachePlugin + decacheModule } // --------------------------------------------------------------------------- diff --git a/server/core/lib/plugins/plugin-manager.ts b/server/core/lib/plugins/plugin-manager.ts index 66b5c5b1841d..94fd07eaf99f 100644 --- a/server/core/lib/plugins/plugin-manager.ts +++ b/server/core/lib/plugins/plugin-manager.ts @@ -1,6 +1,7 @@ import express from 'express' -import { createReadStream, createWriteStream } from 'fs' -import { ensureDir, outputFile, readJSON } from 'fs-extra/esm' +import { createReadStream, createWriteStream, statSync } from 'fs' +import { mkdir, readlink } from 'fs/promises' +import { ensureDir, outputFile, readJSON, ensureSymlink } from 'fs-extra/esm' import { Server } from 'http' import { createRequire } from 'module' import { basename, join } from 'path' @@ -16,7 +17,6 @@ import { ServerHook, ServerHookName } from '@peertube/peertube-models' -import { decachePlugin } from '@server/helpers/decache.js' import { ApplicationModel } from '@server/models/application/application.js' import { MOAuthTokenUser, MUser } from '@server/types/models/index.js' import { isLibraryCodeValid, isPackageJSONValid } from '../../helpers/custom-validators/plugins.js' @@ -77,13 +77,25 @@ export class PluginManager implements ServerHook { private hooks: { [name: string]: HookInformationValue[] } = {} private translations: PluginLocalesTranslations = {} + private readonly latestDirectory = join(CONFIG.STORAGE.PLUGINS_DIR, 'latest') + private server: Server private constructor () { } - init (server: Server) { + async init (server: Server) { this.server = server + + try { + statSync(this.latestDirectory) + } catch (err) { + const workingDir = join(CONFIG.STORAGE.PLUGINS_DIR, Date.now().toString()) + await mkdir(workingDir) + await ensureSymlink(workingDir, this.latestDirectory) + // await writeJSON(join(this.latestDirectory, 'package.json'), {}) + // await writeFile(join(this.latestDirectory, 'yarn.lock'), '') + } } registerWebSocketRouter () { @@ -374,6 +386,11 @@ export class PluginManager implements ServerHook { logger.info('Successful installation of plugin %s.', toInstall) if (register) { + // Unregister old hooks if it's an update + try { + await this.unregister(npmName) + } catch (err) {} + await this.registerPluginOrTheme(plugin) } } catch (rootErr) { @@ -394,6 +411,11 @@ export class PluginManager implements ServerHook { } throw rootErr + } finally { + // Update plugin paths + for (const npmName in this.registeredPlugins) { + this.registeredPlugins[npmName].path = await this.getPluginPath(this.registeredPlugins[npmName].name, this.registeredPlugins[npmName].type) + } } return plugin @@ -411,9 +433,6 @@ export class PluginManager implements ServerHook { version = plugin.latestVersion } - // Unregister old hooks - await this.unregister(npmName) - return this.install({ toInstall: toUpdate, version, fromDisk }) } @@ -463,7 +482,7 @@ export class PluginManager implements ServerHook { logger.info('Registering plugin or theme %s.', npmName) const packageJSON = await this.getPackageJSON(plugin.name, plugin.type) - const pluginPath = this.getPluginPath(plugin.name, plugin.type) + const pluginPath = await this.getPluginPath(plugin.name, plugin.type) this.sanitizeAndCheckPackageJSONOrThrow(packageJSON, plugin.type) @@ -503,9 +522,7 @@ export class PluginManager implements ServerHook { private async registerPlugin (plugin: PluginModel, pluginPath: string, packageJSON: PluginPackageJSON) { const npmName = PluginModel.buildNpmName(plugin.name, plugin.type) - // Delete cache if needed const modulePath = join(pluginPath, packageJSON.library) - decachePlugin(require, modulePath) const library: PluginLibrary = require(modulePath) if (!isLibraryCodeValid(library)) { @@ -530,7 +547,7 @@ export class PluginManager implements ServerHook { private async addTranslations (plugin: PluginModel, npmName: string, translationPaths: PluginTranslationPathsJSON) { for (const locale of Object.keys(translationPaths)) { const path = translationPaths[locale] - const json = await readJSON(join(this.getPluginPath(plugin.name, plugin.type), path)) + const json = await readJSON(join(await this.getPluginPath(plugin.name, plugin.type), path)) const completeLocale = getCompleteLocale(locale) @@ -591,16 +608,17 @@ export class PluginManager implements ServerHook { } } - private getPackageJSON (pluginName: string, pluginType: PluginType_Type) { - const pluginPath = join(this.getPluginPath(pluginName, pluginType), 'package.json') + private async getPackageJSON (pluginName: string, pluginType: PluginType_Type) { + const pluginPath = join(await this.getPluginPath(pluginName, pluginType), 'package.json') return readJSON(pluginPath) as Promise } - private getPluginPath (pluginName: string, pluginType: PluginType_Type) { + private async getPluginPath (pluginName: string, pluginType: PluginType_Type) { const npmName = PluginModel.buildNpmName(pluginName, pluginType) + const currentDirectory = await readlink(join(CONFIG.STORAGE.PLUGINS_DIR, 'latest')) - return join(CONFIG.STORAGE.PLUGINS_DIR, 'node_modules', npmName) + return join(currentDirectory, 'node_modules', npmName) } private getAuth (npmName: string, authName: string) { diff --git a/server/core/lib/plugins/yarn.ts b/server/core/lib/plugins/yarn.ts index 470880f44390..17acee30878e 100644 --- a/server/core/lib/plugins/yarn.ts +++ b/server/core/lib/plugins/yarn.ts @@ -1,6 +1,7 @@ -import { outputJSON, pathExists } from 'fs-extra/esm' +import { copy, ensureSymlink, remove, outputJSON, pathExists } from 'fs-extra/esm' +import { mkdir, readlink } from 'fs/promises' import { join } from 'path' -import { execShell } from '../../helpers/core-utils.js' +import { execShell, getContentHash } from '../../helpers/core-utils.js' import { isNpmPluginNameValid, isPluginStableOrUnstableVersionValid } from '../../helpers/custom-validators/plugins.js' import { logger } from '../../helpers/logger.js' import { CONFIG } from '../../initializers/config.js' @@ -16,7 +17,7 @@ async function installNpmPlugin (npmName: string, versionArg?: string) { let toInstall = npmName if (version) toInstall += `@${version}` - const { stdout } = await execYarn('add ' + toInstall) + const stdout = await execYarn('add ' + toInstall) logger.debug('Added a yarn package.', { yarnStdout: stdout }) } @@ -47,21 +48,55 @@ export { // ############################################################################ async function execYarn (command: string) { + const latestDirectory = join(CONFIG.STORAGE.PLUGINS_DIR, 'latest') + const currentDirectory = await readlink(latestDirectory) + let workingDirectory: string + let stdout: string + try { - const pluginDirectory = CONFIG.STORAGE.PLUGINS_DIR - const pluginPackageJSON = join(pluginDirectory, 'package.json') + const pluginPackageJSON = join(currentDirectory, 'package.json') // Create empty package.json file if needed if (!await pathExists(pluginPackageJSON)) { await outputJSON(pluginPackageJSON, {}) } - return execShell(`yarn ${command}`, { cwd: pluginDirectory }) + const hash = await getContentHash(pluginPackageJSON) + + workingDirectory = join(CONFIG.STORAGE.PLUGINS_DIR, hash) + await mkdir(workingDirectory) + await copy(join(currentDirectory, 'package.json'), join(workingDirectory, 'package.json')) + + try { + await copy(join(currentDirectory, 'yarn.lock'), join(workingDirectory, 'yarn.lock')) + } catch (err) { + logger.debug('No yarn.lock file to copy, will continue without.') + } + + const result = await execShell(`yarn ${command}`, { cwd: workingDirectory }) + stdout = result.stdout } catch (result) { - logger.error('Cannot exec yarn.', { command, err: result.err, stderr: result.stderr }) + logger.error('Cannot exec yarn.', { command, err: result, stderr: result.stderr }) + + await remove(workingDirectory) throw result.err } + + try { + await remove(latestDirectory) + await ensureSymlink(workingDirectory, latestDirectory) + } catch (err) { + logger.error('Cannot create symlink for new plugin set. Trying to restore the old one.', { err }) + await ensureSymlink(currentDirectory, latestDirectory) + logger.info('Succeeded to restore old plugin set.') + + throw err + } + + await remove(currentDirectory) + + return stdout } function checkNpmPluginNameOrThrow (name: string) { diff --git a/server/server.ts b/server/server.ts index 23e6cb1e6138..4b7e829f07a0 100644 --- a/server/server.ts +++ b/server/server.ts @@ -335,7 +335,7 @@ async function startApplication () { OpenTelemetryMetrics.Instance.registerMetrics({ trackerServer }) - PluginManager.Instance.init(server) + await PluginManager.Instance.init(server) // Before PeerTubeSocket init PluginManager.Instance.registerWebSocketRouter()