Skip to content

Commit

Permalink
chore: refactor opt.pg*/$PG* handling
Browse files Browse the repository at this point in the history
  • Loading branch information
derhuerst committed Jul 15, 2024
1 parent 96c8b49 commit e93860f
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 26 deletions.
33 changes: 8 additions & 25 deletions import.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import {
digestFile,
pSpawn,
formatDbName,
getPgConfig,
getPgEnv,
connectToMetaDatabase,
readImportedDatabases,
} from './index.js'
Expand All @@ -29,7 +31,6 @@ const importGtfsAtomically = async (cfg) => {
connectDownloadScriptToStdout,
importScriptVerbose,
connectImportScriptToStdout,
pgHost, pgPort, pgUser, pgPassword, pgMetaDatabase, pgOpts,
databaseNamePrefix,
schemaName,
pathToImportScript,
Expand All @@ -47,12 +48,6 @@ const importGtfsAtomically = async (cfg) => {
connectDownloadScriptToStdout: true,
importScriptVerbose: true,
connectImportScriptToStdout: true,
pgHost: null,
pgPort: null,
pgUser: null,
pgPassword: null,
pgMetaDatabase: process.env.PGDATABASE || null,
pgOpts: {},
schemaName: process.env.GTFS_IMPORTER_SCHEMA || null,
pathToImportScript: PATH_TO_IMPORT_SCRIPT,
pathToDownloadScript: PATH_TO_DOWNLOAD_SCRIPT,
Expand Down Expand Up @@ -99,16 +94,15 @@ const importGtfsAtomically = async (cfg) => {
})
result.downloadDurationMs = performance.now() - _t0Download

const pgConfig = await getPgConfig(cfg)
const pgEnv = getPgEnv(pgConfig)

// `CREATE/DROP DATABASE` can't be run within the transation, so we need need a separate client for it.
// Thus, a newly created database also won't be removed if the transaction fails or is aborted, so we
// have to drop it manually when cleaning up failed/aborted imports.
const dbMngmtClient = await connectToMetaDatabase({
pgHost, pgPort, pgUser, pgPassword, pgMetaDatabase, pgOpts,
})
const dbMngmtClient = await connectToMetaDatabase(cfg)

const client = await connectToMetaDatabase({
pgHost, pgPort, pgUser, pgPassword, pgMetaDatabase, pgOpts,
})
const client = await connectToMetaDatabase(cfg)

// We only ever keep one row in `latest_import`, which contains NULL in the beginning.
await client.query(`\
Expand Down Expand Up @@ -191,23 +185,12 @@ const importGtfsAtomically = async (cfg) => {
logger.info(`importing data into "${dbName}"`)
const _importEnv = {
...process.env,
pgEnv,
PATH: NPM_BIN_DIR + ':' + process.env.PATH,
PGDATABASE: dbName,
GTFS_TMP_DIR: tmpDir,
GTFS_IMPORTER_VERBOSE: importScriptVerbose ? 'true' : 'false',
}
if (pgHost !== null) {
_importEnv.PGHOST = pgHost
}
if (pgPort !== null) {
_importEnv.PGPORT = pgPort
}
if (pgUser !== null) {
_importEnv.PGUSER = pgUser
}
if (pgPassword !== null) {
_importEnv.PGPASSWORD = pgPassword
}
if (schemaName !== null) {
_importEnv.GTFS_IMPORTER_SCHEMA = schemaName
}
Expand Down
34 changes: 33 additions & 1 deletion index.js
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ deepStrictEqual(
{name: 'gtfs_nyct_subway_1712169379_0f1deb', importedAt: 1712169379, feedDigest: '0f1deb'},
)

const connectToMetaDatabase = async (cfg) => {
const getPgConfig = async (cfg) => {
const {
pgHost,
pgPort,
Expand Down Expand Up @@ -113,12 +113,42 @@ const connectToMetaDatabase = async (cfg) => {
pgConfig.database = process.env.PGDATABASE
}

return pgConfig
}

const connectToMetaDatabase = async (cfg) => {
const pgConfig = await getPgConfig(cfg)
const db = new Client(pgConfig)
await db.connect()

return db
}

// https://www.postgresql.org/docs/15/libpq-connect.html#id-1.7.3.8.3.5
const getPgEnv = async (pgConfig) => {
const pgEnv = {
}

if (pgConfig.host !== null) {
pgEnv.PGHOST = pgConfig.host
}
if (pgConfig.port !== null) {
pgEnv.PGPORT = pgConfig.port
}
if (pgConfig.user !== null) {
pgEnv.PGUSER = pgConfig.user
}
if (pgConfig.password !== null) {
pgEnv.PGPASSWORD = pgConfig.password
}
if (pgConfig.database !== null) {
pgEnv.PGDATABASE = pgConfig.database
}
// todo: ssl mode?

return pgConfig
}

const readImportedDatabases = async (cfg) => {
const {
databaseNamePrefix,
Expand Down Expand Up @@ -175,6 +205,8 @@ export {
pSpawn,
formatDbName,
parseDbName,
getPgEnv,
getPgConfig,
connectToMetaDatabase,
readImportedDatabases,
}

0 comments on commit e93860f

Please sign in to comment.