Merge pull request #4956 from mnaamani/nara-master-update

Update nara from master

.github/workflows/joystream-apps-docker.yml

@@ -2,6 +2,11 @@ name: Publish Colossus/Argus/QueryNode Docker images
 
 on:
   workflow_dispatch:
+    inputs:
+      packages:
+        description: 'Comma-separated list of packages to publish to Docker Hub'
+        required: true
+        default: '*'
   pull_request:
 
 jobs:
@@ -42,6 +47,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Build joystream-node
+        if: github.event_name == 'pull_request'
         run: |
           RUNTIME_PROFILE=TESTING ./build-node-docker.sh
           # reclaim some disk space
@@ -86,26 +92,38 @@ jobs:
           tags: joystream/query-node:latest
 
       - name: Test with latest images
+        if: github.event_name == 'pull_request'
         run: |
           docker system prune --force
           yarn build:packages
           cp docker-compose-no-bind-volumes.yml docker-compose.yml
           ./tests/network-tests/run-tests.sh content-directory
 
-      - name: Push new versions
-        if: github.event_name == 'workflow_dispatch'
+      - name: Push storage-node
+        if: contains(github.event.inputs.packages, 'storage-node') || github.event.inputs.packages == '*'
         run: |
           docker image tag joystream/storage-node:latest joystream/storage-node:${{ steps.extract_versions.outputs.colossus_version }}
-          docker image tag joystream/distributor-node:latest joystream/distributor-node:${{ steps.extract_versions.outputs.argus_version }}
-          docker image tag joystream/query-node:latest joystream/query-node:${{ steps.extract_versions.outputs.qn_version }}
           docker push joystream/storage-node:${{ steps.extract_versions.outputs.colossus_version }}
+      - name: Push distributor-node
+        if: contains(github.event.inputs.packages, 'distributor-node') || github.event.inputs.packages == '*'
+        run: |
+          docker image tag joystream/distributor-node:latest joystream/distributor-node:${{ steps.extract_versions.outputs.argus_version }}
           docker push joystream/distributor-node:${{ steps.extract_versions.outputs.argus_version }}
+      - name: Push query-node
+        if: contains(github.event.inputs.packages, 'query-node') || github.event.inputs.packages == '*'
+        run: |
+          docker image tag joystream/query-node:latest joystream/query-node:${{ steps.extract_versions.outputs.qn_version }}
           docker push joystream/query-node:${{ steps.extract_versions.outputs.qn_version }}
 
-      - name: Push latest tag
-        if: github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch'
-        run: |
-          docker push joystream/storage-node:latest
-          docker push joystream/distributor-node:latest
-          docker push joystream/query-node:latest
+      - name: Push storage-node latest tag
+        if: github.ref == 'refs/heads/master' && (contains(github.event.inputs.packages, 'storage-node') || github.event.inputs.packages == '*')
+        run: docker push joystream/storage-node:latest
+
+      - name: Push distributor-node latest tag
+        if: github.ref == 'refs/heads/master' && (contains(github.event.inputs.packages, 'distributor-node') || github.event.inputs.packages == '*')
+        run: docker push joystream/distributor-node:latest
+
+      - name: Push query-node latest tag
+        if: github.ref == 'refs/heads/master' && (contains(github.event.inputs.packages, 'query-node') || github.event.inputs.packages == '*')
+        run: docker push joystream/query-node:latest
 

cli/package.json

@@ -56,7 +56,7 @@
     "proper-lockfile": "^4.1.1",
     "slug": "^2.1.1",
     "tslib": "^1.11.1",
-    "@joystream/js": "^1.4.0"
+    "@joystream/js": "^1.6.0"
   },
   "devDependencies": {
     "@graphql-codegen/cli": "^1.21.4",

distributor-node/CHANGELOG.md

@@ -1,3 +1,22 @@
+##
+
+### 1.4.0
+
+- Add 1s delay between fetching pages for QN state sync. This will allow QN to have some breathing space and process other requests that may have been stuck because of heavy processing.
+- Add `Access-Control-Expose-Headers` response header to allow web clients checking cache status of downloaded file.
+- Include response headers in `http` logs
+- Disable open-api express response validation if NODE_ENV is set to 'production' or 'prod'. This should improve response times when serving assets.
+- Include `nodeEnv` in `/api/v1/status` response, to help detect mis-configured nodes.
+- **FIX** Axios Error Logging: Logging the error, when asset download from storage-node time outs, has been fixed to include the _only_ error message, response, status code and bunch of other fields. Previously, logging error object (which includes axios client instance), failed with `Converting circular structure to JSON` error and causing the distributor-node to crash.
+
+### 1.3.1
+
+- **FIX** QN state sync: The QN state sync that runs on startup and on interval, has been split to multiple paginated queries so that it doesn't crash QN's GraphQL server because of huge payload: [#4921](https://github.com/Joystream/joystream/pull/4921)
+
+### 1.3.0
+
+- Adds support for TTL based caching of `StorageDataObject` QN entity for `HEAD /assets` requests. The TTL is configurable using `interval.queryNodeCacheTTL` flag.
+
 ### 1.2.2
 
 - **FIX** `sendExtrinsic`: The send extrinsic function (which is a wrapper around PolkadotJS `tx.signAndSend` function) has been fixed to handle the case when tx has been finalized before the callback registered in `tx.signAndSend` would run.

distributor-node/client/api.ts

@@ -126,6 +126,12 @@ export interface StatusResponse {
      * @memberof StatusResponse
      */
     'downloadsInProgress': number;
+    /**
+     * 
+     * @type {string}
+     * @memberof StatusResponse
+     */
+    'nodeEnv': string;
     /**
      * 
      * @type {StatusResponseQueryNodeStatus}

distributor-node/client/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@joystream/distributor-node-client",
   "description": "Joystream autogenerated distributor node api client.",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "author": "Joystream contributors",
   "bugs": "https://github.com/Joystream/joystream/issues",
   "dependencies": {

distributor-node/config.yml

@@ -26,6 +26,7 @@ limits:
   outboundRequestsTimeoutMs: 5000
   pendingDownloadTimeoutSec: 3600
   maxCachedItemSize: 1G
+  queryNodeCacheTTL: 60
 intervals:
   saveCacheState: 60
   checkStorageNodeResponseTimes: 60

distributor-node/docs/api/public/index.md

@@ -85,6 +85,7 @@ Returns json object describing current node status.
   "storageUsed": 0,
   "uptime": 0,
   "downloadsInProgress": 0,
+  "nodeEnv": "string",
   "queryNodeStatus": {
     "url": "string",
     "chainHead": 0,
@@ -340,6 +341,7 @@ This operation does not require authentication
   "storageUsed": 0,
   "uptime": 0,
   "downloadsInProgress": 0,
+  "nodeEnv": "string",
   "queryNodeStatus": {
     "url": "string",
     "chainHead": 0,
@@ -361,6 +363,7 @@ This operation does not require authentication
 |storageUsed|integer|true|none|none|
 |uptime|integer|true|none|none|
 |downloadsInProgress|integer|true|none|none|
+|nodeEnv|string|true|none|none|
 |queryNodeStatus|object|true|none|none|
 |» url|string|true|none|none|
 |» chainHead|integer|true|none|none|

distributor-node/docs/schema/definition-properties-limits-properties-querynodecachettl.md

@@ -0,0 +1,15 @@
+## queryNodeCacheTTL Type
+
+`integer`
+
+## queryNodeCacheTTL Constraints
+
+**minimum**: the value of this number must greater than or equal to: `1`
+
+## queryNodeCacheTTL Default Value
+
+The default value is:
+
+```json
+60
+```

distributor-node/docs/schema/definition-properties-limits.md

@@ -13,6 +13,7 @@
 | [pendingDownloadTimeoutSec](#pendingdownloadtimeoutsec)                 | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-pendingdownloadtimeoutsec.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/pendingDownloadTimeoutSec")                 |
 | [maxCachedItemSize](#maxcacheditemsize)                                 | `string`  | Optional | cannot be null | [Distributor node configuration](definition-properties-limits-properties-maxcacheditemsize.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/maxCachedItemSize")                                 |
 | [dataObjectSourceByObjectIdTTL](#dataobjectsourcebyobjectidttl)         | `integer` | Optional | cannot be null | [Distributor node configuration](definition-properties-limits-properties-dataobjectsourcebyobjectidttl.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/dataObjectSourceByObjectIdTTL")         |
+| [queryNodeCacheTTL](#querynodecachettl)                                 | `integer` | Optional | cannot be null | [Distributor node configuration](definition-properties-limits-properties-querynodecachettl.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/queryNodeCacheTTL")                                 |
 
 ## storage
 
@@ -187,3 +188,33 @@ The default value is:
 ```json
 60
 ```
+
+## queryNodeCacheTTL
+
+TTL (in seconds) for the Apollo's InMemoryCache, to cache the data fetched from the query node.
+
+`queryNodeCacheTTL`
+
+*   is optional
+
+*   Type: `integer`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-limits-properties-querynodecachettl.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/queryNodeCacheTTL")
+
+### queryNodeCacheTTL Type
+
+`integer`
+
+### queryNodeCacheTTL Constraints
+
+**minimum**: the value of this number must greater than or equal to: `1`
+
+### queryNodeCacheTTL Default Value
+
+The default value is:
+
+```json
+60
+```

distributor-node/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@joystream/distributor-cli",
   "description": "Joystream distributor node CLI",
-  "version": "1.2.2",
+  "version": "1.4.0",
   "author": "Joystream contributors",
   "bin": {
     "joystream-distributor": "./bin/run"
@@ -14,6 +14,7 @@
     "@joystream/opentelemetry": "1.0.0",
     "@joystream/storage-node-client": "^3.0.0",
     "@joystream/types": "^4.0.0",
+    "@nerdwallet/apollo-cache-policies": "2.10.0",
     "@oclif/command": "^1",
     "@oclif/config": "^1",
     "@oclif/plugin-help": "^3",
@@ -66,12 +67,12 @@
     "@types/cors": "^2.8.12",
     "@types/express-http-proxy": "^1.6.2",
     "@types/inquirer": "^8.1.1",
+    "@types/mime": "^3.0.1",
     "@types/mocha": "^5",
     "@types/node": "^14",
     "@types/node-cache": "^4.2.5",
     "@types/node-cleanup": "^2.1.1",
     "@types/send": "^0.17.0",
-    "@types/mime": "^3.0.1",
     "@types/ws": "^5.1.2",
     "chai": "^4",
     "globby": "^10",

distributor-node/src/api-spec/public.yml

@@ -166,6 +166,7 @@ components:
         - storageUsed
         - uptime
         - downloadsInProgress
+        - nodeEnv
         - queryNodeStatus
       properties:
         id:
@@ -189,6 +190,8 @@ components:
         downloadsInProgress:
           type: integer
           minimum: 0
+        nodeEnv:
+          type: string
         queryNodeStatus:
           type: object
           required:

distributor-node/src/schemas/configSchema.ts

@@ -165,6 +165,12 @@ export const configSchema: JSONSchema4 = objectSchema({
           type: 'integer',
           minimum: 1,
         },
+        queryNodeCacheTTL: {
+          description: ` TTL (in seconds) for the Apollo's InMemoryCache, to cache the data fetched from the query node.`,
+          default: 60,
+          type: 'integer',
+          minimum: 1,
+        },
       },
       required: [
         'storage',

distributor-node/src/services/content/ContentService.ts

@@ -10,6 +10,7 @@ import { NetworkingService } from '../networking'
 import { ContentHash } from '../crypto/ContentHash'
 import { PendingDownloadStatusType } from '../networking/PendingDownload'
 import { FSP } from './FSPromise'
+import { QueryFetchPolicy } from '../networking/query-node/api'
 
 export const DEFAULT_CONTENT_TYPE = 'application/octet-stream'
 export const MIME_TYPE_DETECTION_CHUNK_SIZE = 4100
@@ -313,7 +314,7 @@ export class ContentService {
     })
   }
 
-  public async objectStatus(objectId: string): Promise<ObjectStatus> {
+  public async objectStatus(objectId: string, qnFetchPolicy: QueryFetchPolicy = 'no-cache'): Promise<ObjectStatus> {
     const pendingDownload = this.stateCache.getPendingDownload(objectId)
 
     if (!pendingDownload && this.exists(objectId)) {
@@ -324,7 +325,7 @@ export class ContentService {
       return { type: ObjectStatusType.PendingDownload, pendingDownload }
     }
 
-    const objectInfo = await this.networking.dataObjectInfo(objectId)
+    const objectInfo = await this.networking.dataObjectInfo(objectId, qnFetchPolicy)
     if (!objectInfo.exists) {
       return { type: ObjectStatusType.NotFound }
     }

distributor-node/src/services/httpApi/HttpApiBase.ts

@@ -48,8 +48,10 @@ export abstract class HttpApiBase {
   protected abstract routes(): HttpApiRoute[]
 
   protected defaultOpenApiValidatorConfig(): Partial<OpenApiValidatorOpts> {
-    const isProd = process.env.NODE_ENV === 'prod'
+    // The convention is 'production' but we allow short form also 'prod'
+    const isProd = process.env.NODE_ENV === 'prod' || process.env.NODE_ENV === 'production'
     return {
+      // Validating responses severely impacts latency when reponses size is large!
       validateResponses: !isProd,
     }
   }
@@ -61,7 +63,10 @@ export abstract class HttpApiBase {
       winstonInstance: this.logger,
       level: 'http',
       dynamicMeta: (req, res) => {
-        return { prematurelyClosed: res.locals.prematurelyClosed ?? false }
+        return {
+          prematurelyClosed: res.locals.prematurelyClosed ?? false,
+          res: { headers: res.getHeaders(), statusCode: res.statusCode },
+        }
       },
     }
   }

distributor-node/src/services/httpApi/controllers/public.ts

@@ -219,11 +219,12 @@ export class PublicApiController {
 
   public async assetHead(req: express.Request<AssetRouteParams>, res: express.Response): Promise<void> {
     const { objectId } = req.params
-    const objectStatus = await this.content.objectStatus(objectId)
+    const objectStatus = await this.content.objectStatus(objectId, 'cache-first')
 
     res.setHeader('timing-allow-origin', '*')
     res.setHeader('accept-ranges', 'bytes')
     res.setHeader('content-disposition', 'inline')
+    res.setHeader('access-control-expose-headers', 'x-cache, x-data-source')
 
     switch (objectStatus.type) {
       case ObjectStatusType.Available:
@@ -271,6 +272,7 @@ export class PublicApiController {
     })
 
     res.setHeader('timing-allow-origin', '*')
+    res.setHeader('access-control-expose-headers', 'x-cache, x-data-source')
 
     switch (objectStatus.type) {
       case ObjectStatusType.Available:
@@ -303,6 +305,7 @@ export class PublicApiController {
       uptime: Math.floor(process.uptime()),
       downloadsInProgress: this.stateCache.getPendingDownloadsCount(),
       queryNodeStatus: await this.networking.getQueryNodeStatus(),
+      nodeEnv: process.env.NODE_ENV || '',
     }
     res.status(200).json(data)
   }