support setting pypiBaseUrl by environment variables and queryParamet…

…ers; affects [pypi] (#10044) * support setting pypiBaseUrl by environment variables * Add support for pypiBaseUrl configuration * Update Pypi services to include pypiBaseUrl parameter * change package name example to a more well-known package * Update custom-environment-variables.yml * Update Pypi services to include pypiBaseUrl parameter * fix openapi mismatch * Update doc/server-secrets.md --------- Co-authored-by: chris48s <[email protected]>
badges · Apr 21, 2024 · e8671be · e8671be
1 parent 1e5d03f
commit e8671be
Show file tree

Hide file tree

Showing 14 changed files with 78 additions and 69 deletions.
diff --git a/config/custom-environment-variables.yml b/config/custom-environment-variables.yml
@@ -52,6 +52,8 @@ public:
       authorizedOrigins: 'NPM_ORIGINS'
     obs:
       authorizedOrigins: 'OBS_ORIGINS'
+    pypi:
+      baseUri: 'PYPI_URL'
     sonar:
       authorizedOrigins: 'SONAR_ORIGINS'
     teamcity:

diff --git a/config/default.yml b/config/default.yml
@@ -22,6 +22,8 @@ public:
       restApiVersion: '2022-11-28'
     obs:
       authorizedOrigins: 'https://api.opensuse.org'
+    pypi:
+      baseUri: 'https://pypi.org'
     weblate:
       authorizedOrigins: 'https://hosted.weblate.org'
     trace: false

diff --git a/core/server/server.js b/core/server/server.js
@@ -139,6 +139,9 @@ const publicConfigSchema = Joi.object({
     nexus: defaultService,
     npm: defaultService,
     obs: defaultService,
+    pypi: {
+      baseUri: requiredUrl,
+    },
     sonar: defaultService,
     teamcity: defaultService,
     weblate: defaultService,

diff --git a/doc/server-secrets.md b/doc/server-secrets.md
@@ -283,6 +283,13 @@ The Pepy API requires authentication. To obtain a key,
 Create an account, sign in and obtain generate a key on your
 [account page](https://www.pepy.tech/user).
 
+### PyPI
+
+- `PYPI_URL` (yml: `public.pypi.baseUri`)
+
+`PYPI_URL` can be used to optionally send all the PyPI requests to a Self-hosted Pypi registry,
+users can also override this by query parameter `pypiBaseUrl`.
+
 ### SymfonyInsight (formerly Sensiolabs)
 
 - `SL_INSIGHT_USER_UUID` (yml: `private.sl_insight_userUuid`)

diff --git a/services/pypi/pypi-base.js b/services/pypi/pypi-base.js
@@ -1,5 +1,7 @@
 import Joi from 'joi'
-import { BaseJsonService } from '../index.js'
+import config from 'config'
+import { optionalUrl } from '../validators.js'
+import { BaseJsonService, queryParam, pathParam } from '../index.js'
 
 const schema = Joi.object({
   info: Joi.object({
@@ -18,18 +20,38 @@ const schema = Joi.object({
     .required(),
 }).required()
 
+export const queryParamSchema = Joi.object({
+  pypiBaseUrl: optionalUrl,
+}).required()
+
+export const pypiPackageParam = pathParam({
+  name: 'packageName',
+  example: 'Django',
+})
+
+export const pypiBaseUrlParam = queryParam({
+  name: 'pypiBaseUrl',
+  example: 'https://pypi.org',
+})
+
+export const pypiGeneralParams = [pypiPackageParam, pypiBaseUrlParam]
+
 export default class PypiBase extends BaseJsonService {
   static buildRoute(base) {
     return {
       base,
       pattern: ':egg+',
+      queryParamSchema,
     }
   }
 
-  async fetch({ egg }) {
+  async fetch({ egg, pypiBaseUrl = null }) {
+    const defaultpypiBaseUrl =
+      config.util.toObject().public.services.pypi.baseUri
+    pypiBaseUrl = pypiBaseUrl || defaultpypiBaseUrl
     return this._requestJson({
       schema,
-      url: `https://pypi.org/pypi/${egg}/json`,
+      url: `${pypiBaseUrl}/pypi/${egg}/json`,
       httpErrors: { 404: 'package or version not found' },
     })
   }

diff --git a/services/pypi/pypi-downloads.service.js b/services/pypi/pypi-downloads.service.js
@@ -1,7 +1,8 @@
 import Joi from 'joi'
 import { nonNegativeInteger } from '../validators.js'
-import { BaseJsonService, pathParams } from '../index.js'
+import { BaseJsonService, pathParam } from '../index.js'
 import { renderDownloadsBadge } from '../downloads.js'
+import { pypiPackageParam } from './pypi-base.js'
 
 const schema = Joi.object({
   data: Joi.object({
@@ -42,15 +43,15 @@ export default class PypiDownloads extends BaseJsonService {
         summary: 'PyPI - Downloads',
         description:
           'Python package downloads from [pypistats](https://pypistats.org/)',
-        parameters: pathParams(
-          {
+        parameters: [
+          pathParam({
             name: 'period',
             example: 'dd',
             schema: { type: 'string', enum: this.getEnum('period') },
             description: 'Daily, Weekly, or Monthly downloads',
-          },
-          { name: 'packageName', example: 'Django' },
-        ),
+          }),
+          pypiPackageParam,
+        ],
       },
     },
   }

diff --git a/services/pypi/pypi-format.service.js b/services/pypi/pypi-format.service.js
@@ -1,5 +1,4 @@
-import { pathParams } from '../index.js'
-import PypiBase from './pypi-base.js'
+import PypiBase, { pypiGeneralParams } from './pypi-base.js'
 import { getPackageFormats } from './pypi-helpers.js'
 
 export default class PypiFormat extends PypiBase {
@@ -11,10 +10,7 @@ export default class PypiFormat extends PypiBase {
     '/pypi/format/{packageName}': {
       get: {
         summary: 'PyPI - Format',
-        parameters: pathParams({
-          name: 'packageName',
-          example: 'Django',
-        }),
+        parameters: pypiGeneralParams,
       },
     },
   }
@@ -40,8 +36,8 @@ export default class PypiFormat extends PypiBase {
     }
   }
 
-  async handle({ egg }) {
-    const packageData = await this.fetch({ egg })
+  async handle({ egg }, { pypiBaseUrl }) {
+    const packageData = await this.fetch({ egg, pypiBaseUrl })
     const { hasWheel, hasEgg } = getPackageFormats(packageData)
     return this.constructor.render({ hasWheel, hasEgg })
   }

diff --git a/services/pypi/pypi-framework-versions.service.js b/services/pypi/pypi-framework-versions.service.js
@@ -1,5 +1,5 @@
 import { InvalidResponse, pathParams } from '../index.js'
-import PypiBase from './pypi-base.js'
+import PypiBase, { pypiBaseUrlParam } from './pypi-base.js'
 import { sortPypiVersions, parseClassifiers } from './pypi-helpers.js'
 
 const frameworkNameMap = {
@@ -63,7 +63,7 @@ export default class PypiFrameworkVersion extends PypiBase {
             schema: { type: 'string', enum: Object.keys(frameworkNameMap) },
           },
           { name: 'packageName', example: 'plone.volto' },
-        ),
+        ).concat(pypiBaseUrlParam),
       },
     },
   }
@@ -80,15 +80,15 @@ export default class PypiFrameworkVersion extends PypiBase {
     }
   }
 
-  async handle({ frameworkName, packageName }) {
+  async handle({ frameworkName, packageName }, { pypiBaseUrl }) {
     const classifier = frameworkNameMap[frameworkName]
       ? frameworkNameMap[frameworkName].classifier
       : frameworkName
     const name = frameworkNameMap[frameworkName]
       ? frameworkNameMap[frameworkName].name
       : frameworkName
     const regex = new RegExp(`^Framework :: ${classifier} :: ([\\d.]+)$`)
-    const packageData = await this.fetch({ egg: packageName })
+    const packageData = await this.fetch({ egg: packageName, pypiBaseUrl })
     const versions = parseClassifiers(packageData, regex)
 
     if (versions.length === 0) {

diff --git a/services/pypi/pypi-implementation.service.js b/services/pypi/pypi-implementation.service.js
@@ -1,5 +1,4 @@
-import { pathParams } from '../index.js'
-import PypiBase from './pypi-base.js'
+import PypiBase, { pypiGeneralParams } from './pypi-base.js'
 import { parseClassifiers } from './pypi-helpers.js'
 
 export default class PypiImplementation extends PypiBase {
@@ -11,10 +10,7 @@ export default class PypiImplementation extends PypiBase {
     '/pypi/implementation/{packageName}': {
       get: {
         summary: 'PyPI - Implementation',
-        parameters: pathParams({
-          name: 'packageName',
-          example: 'Django',
-        }),
+        parameters: pypiGeneralParams,
       },
     },
   }
@@ -28,8 +24,8 @@ export default class PypiImplementation extends PypiBase {
     }
   }
 
-  async handle({ egg }) {
-    const packageData = await this.fetch({ egg })
+  async handle({ egg }, { pypiBaseUrl }) {
+    const packageData = await this.fetch({ egg, pypiBaseUrl })
 
     let implementations = parseClassifiers(
       packageData,

diff --git a/services/pypi/pypi-license.service.js b/services/pypi/pypi-license.service.js
@@ -1,6 +1,5 @@
-import { pathParams } from '../index.js'
 import { renderLicenseBadge } from '../licenses.js'
-import PypiBase from './pypi-base.js'
+import PypiBase, { pypiGeneralParams } from './pypi-base.js'
 import { getLicenses } from './pypi-helpers.js'
 
 export default class PypiLicense extends PypiBase {
@@ -12,10 +11,7 @@ export default class PypiLicense extends PypiBase {
     '/pypi/l/{packageName}': {
       get: {
         summary: 'PyPI - License',
-        parameters: pathParams({
-          name: 'packageName',
-          example: 'Django',
-        }),
+        parameters: pypiGeneralParams,
       },
     },
   }
@@ -24,8 +20,8 @@ export default class PypiLicense extends PypiBase {
     return renderLicenseBadge({ licenses })
   }
 
-  async handle({ egg }) {
-    const packageData = await this.fetch({ egg })
+  async handle({ egg }, { pypiBaseUrl }) {
+    const packageData = await this.fetch({ egg, pypiBaseUrl })
     const licenses = getLicenses(packageData)
     return this.constructor.render({ licenses })
   }

diff --git a/services/pypi/pypi-python-versions.service.js b/services/pypi/pypi-python-versions.service.js
@@ -1,6 +1,5 @@
 import semver from 'semver'
-import { pathParams } from '../index.js'
-import PypiBase from './pypi-base.js'
+import PypiBase, { pypiGeneralParams } from './pypi-base.js'
 import { parseClassifiers } from './pypi-helpers.js'
 
 export default class PypiPythonVersions extends PypiBase {
@@ -12,10 +11,7 @@ export default class PypiPythonVersions extends PypiBase {
     '/pypi/pyversions/{packageName}': {
       get: {
         summary: 'PyPI - Python Version',
-        parameters: pathParams({
-          name: 'packageName',
-          example: 'Django',
-        }),
+        parameters: pypiGeneralParams,
       },
     },
   }
@@ -48,8 +44,8 @@ export default class PypiPythonVersions extends PypiBase {
     }
   }
 
-  async handle({ egg }) {
-    const packageData = await this.fetch({ egg })
+  async handle({ egg }, { pypiBaseUrl }) {
+    const packageData = await this.fetch({ egg, pypiBaseUrl })
 
     const versions = parseClassifiers(
       packageData,

diff --git a/services/pypi/pypi-status.service.js b/services/pypi/pypi-status.service.js
@@ -1,5 +1,4 @@
-import { pathParams } from '../index.js'
-import PypiBase from './pypi-base.js'
+import PypiBase, { pypiGeneralParams } from './pypi-base.js'
 import { parseClassifiers } from './pypi-helpers.js'
 
 export default class PypiStatus extends PypiBase {
@@ -11,10 +10,7 @@ export default class PypiStatus extends PypiBase {
     '/pypi/status/{packageName}': {
       get: {
         summary: 'PyPI - Status',
-        parameters: pathParams({
-          name: 'packageName',
-          example: 'Django',
-        }),
+        parameters: pypiGeneralParams,
       },
     },
   }
@@ -40,8 +36,8 @@ export default class PypiStatus extends PypiBase {
     }
   }
 
-  async handle({ egg }) {
-    const packageData = await this.fetch({ egg })
+  async handle({ egg }, { pypiBaseUrl }) {
+    const packageData = await this.fetch({ egg, pypiBaseUrl })
 
     // Possible statuses:
     // - Development Status :: 1 - Planning

diff --git a/services/pypi/pypi-version.service.js b/services/pypi/pypi-version.service.js
@@ -1,7 +1,6 @@
-import { pathParams } from '../index.js'
 import { pep440VersionColor } from '../color-formatters.js'
 import { renderVersionBadge } from '../version.js'
-import PypiBase from './pypi-base.js'
+import PypiBase, { pypiGeneralParams } from './pypi-base.js'
 
 export default class PypiVersion extends PypiBase {
   static category = 'version'
@@ -12,10 +11,7 @@ export default class PypiVersion extends PypiBase {
     '/pypi/v/{packageName}': {
       get: {
         summary: 'PyPI - Version',
-        parameters: pathParams({
-          name: 'packageName',
-          example: 'nine',
-        }),
+        parameters: pypiGeneralParams,
       },
     },
   }
@@ -26,10 +22,10 @@ export default class PypiVersion extends PypiBase {
     return renderVersionBadge({ version, versionFormatter: pep440VersionColor })
   }
 
-  async handle({ egg }) {
+  async handle({ egg }, { pypiBaseUrl }) {
     const {
       info: { version },
-    } = await this.fetch({ egg })
+    } = await this.fetch({ egg, pypiBaseUrl })
     return this.constructor.render({ version })
   }
 }
diff --git a/services/pypi/pypi-wheel.service.js b/services/pypi/pypi-wheel.service.js
@@ -1,5 +1,4 @@
-import { pathParams } from '../index.js'
-import PypiBase from './pypi-base.js'
+import PypiBase, { pypiGeneralParams } from './pypi-base.js'
 import { getPackageFormats } from './pypi-helpers.js'
 
 export default class PypiWheel extends PypiBase {
@@ -11,10 +10,7 @@ export default class PypiWheel extends PypiBase {
     '/pypi/wheel/{packageName}': {
       get: {
         summary: 'PyPI - Wheel',
-        parameters: pathParams({
-          name: 'packageName',
-          example: 'Django',
-        }),
+        parameters: pypiGeneralParams,
       },
     },
   }
@@ -35,8 +31,8 @@ export default class PypiWheel extends PypiBase {
     }
   }
 
-  async handle({ egg }) {
-    const packageData = await this.fetch({ egg })
+  async handle({ egg }, { pypiBaseUrl }) {
+    const packageData = await this.fetch({ egg, pypiBaseUrl })
     const { hasWheel } = getPackageFormats(packageData)
     return this.constructor.render({ hasWheel })
   }