From e93860f5d17069175904b05c98e44869856d3d8a Mon Sep 17 00:00:00 2001 From: Jannis R Date: Mon, 15 Jul 2024 17:07:13 +0200 Subject: [PATCH] chore: refactor opt.pg*/$PG* handling --- import.js | 33 ++++++++------------------------- index.js | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/import.js b/import.js index 483aaaa..0db4483 100644 --- a/import.js +++ b/import.js @@ -6,6 +6,8 @@ import { digestFile, pSpawn, formatDbName, + getPgConfig, + getPgEnv, connectToMetaDatabase, readImportedDatabases, } from './index.js' @@ -29,7 +31,6 @@ const importGtfsAtomically = async (cfg) => { connectDownloadScriptToStdout, importScriptVerbose, connectImportScriptToStdout, - pgHost, pgPort, pgUser, pgPassword, pgMetaDatabase, pgOpts, databaseNamePrefix, schemaName, pathToImportScript, @@ -47,12 +48,6 @@ const importGtfsAtomically = async (cfg) => { connectDownloadScriptToStdout: true, importScriptVerbose: true, connectImportScriptToStdout: true, - pgHost: null, - pgPort: null, - pgUser: null, - pgPassword: null, - pgMetaDatabase: process.env.PGDATABASE || null, - pgOpts: {}, schemaName: process.env.GTFS_IMPORTER_SCHEMA || null, pathToImportScript: PATH_TO_IMPORT_SCRIPT, pathToDownloadScript: PATH_TO_DOWNLOAD_SCRIPT, @@ -99,16 +94,15 @@ const importGtfsAtomically = async (cfg) => { }) result.downloadDurationMs = performance.now() - _t0Download + const pgConfig = await getPgConfig(cfg) + const pgEnv = getPgEnv(pgConfig) + // `CREATE/DROP DATABASE` can't be run within the transation, so we need need a separate client for it. // Thus, a newly created database also won't be removed if the transaction fails or is aborted, so we // have to drop it manually when cleaning up failed/aborted imports. - const dbMngmtClient = await connectToMetaDatabase({ - pgHost, pgPort, pgUser, pgPassword, pgMetaDatabase, pgOpts, - }) + const dbMngmtClient = await connectToMetaDatabase(cfg) - const client = await connectToMetaDatabase({ - pgHost, pgPort, pgUser, pgPassword, pgMetaDatabase, pgOpts, - }) + const client = await connectToMetaDatabase(cfg) // We only ever keep one row in `latest_import`, which contains NULL in the beginning. await client.query(`\ @@ -191,23 +185,12 @@ const importGtfsAtomically = async (cfg) => { logger.info(`importing data into "${dbName}"`) const _importEnv = { ...process.env, + pgEnv, PATH: NPM_BIN_DIR + ':' + process.env.PATH, PGDATABASE: dbName, GTFS_TMP_DIR: tmpDir, GTFS_IMPORTER_VERBOSE: importScriptVerbose ? 'true' : 'false', } - if (pgHost !== null) { - _importEnv.PGHOST = pgHost - } - if (pgPort !== null) { - _importEnv.PGPORT = pgPort - } - if (pgUser !== null) { - _importEnv.PGUSER = pgUser - } - if (pgPassword !== null) { - _importEnv.PGPASSWORD = pgPassword - } if (schemaName !== null) { _importEnv.GTFS_IMPORTER_SCHEMA = schemaName } diff --git a/index.js b/index.js index 0823c33..17be2ce 100644 --- a/index.js +++ b/index.js @@ -66,7 +66,7 @@ deepStrictEqual( {name: 'gtfs_nyct_subway_1712169379_0f1deb', importedAt: 1712169379, feedDigest: '0f1deb'}, ) -const connectToMetaDatabase = async (cfg) => { +const getPgConfig = async (cfg) => { const { pgHost, pgPort, @@ -113,12 +113,42 @@ const connectToMetaDatabase = async (cfg) => { pgConfig.database = process.env.PGDATABASE } + return pgConfig +} + +const connectToMetaDatabase = async (cfg) => { + const pgConfig = await getPgConfig(cfg) const db = new Client(pgConfig) await db.connect() return db } +// https://www.postgresql.org/docs/15/libpq-connect.html#id-1.7.3.8.3.5 +const getPgEnv = async (pgConfig) => { + const pgEnv = { + } + + if (pgConfig.host !== null) { + pgEnv.PGHOST = pgConfig.host + } + if (pgConfig.port !== null) { + pgEnv.PGPORT = pgConfig.port + } + if (pgConfig.user !== null) { + pgEnv.PGUSER = pgConfig.user + } + if (pgConfig.password !== null) { + pgEnv.PGPASSWORD = pgConfig.password + } + if (pgConfig.database !== null) { + pgEnv.PGDATABASE = pgConfig.database + } + // todo: ssl mode? + + return pgConfig +} + const readImportedDatabases = async (cfg) => { const { databaseNamePrefix, @@ -175,6 +205,8 @@ export { pSpawn, formatDbName, parseDbName, + getPgEnv, + getPgConfig, connectToMetaDatabase, readImportedDatabases, }