Merge pull request #4965 from mnaamani/nara-master-update

Nara master update

.github/workflows/joystream-node-docker-publish.yml

@@ -4,9 +4,6 @@ name: joystream-node-docker-publish
 on:
   - workflow_dispatch
 
-env:
-  REPOSITORY: joystream/node
-
 jobs:
   main:
     name: Publish joystream/node:latest docker image
@@ -20,11 +17,6 @@ jobs:
         with:
           node-version: '18.x'
 
-      - id: compute_shasum
-        name: Compute runtime code shasum
-        run: |
-          export TAG_SHASUM=`scripts/runtime-code-shasum.sh`
-
       - name: Login to DockerHub
         uses: docker/login-action@v2
         with:
@@ -33,8 +25,9 @@ jobs:
 
       - name: Publish manifest
         run: |
-          IMAGE=${{ env.REPOSITORY }}:${{ env.TAG_SHASUM }}
-          LATEST_TAG=${{ env.REPOSITORY }}:latest
+          SHASUM=`scripts/runtime-code-shasum.sh`
+          IMAGE=joystream/node:$SHASUM
+          LATEST_TAG=joystream/node:latest
           docker pull $IMAGE-amd64
           docker pull $IMAGE-arm64
           docker manifest create $LATEST_TAG $IMAGE-amd64 $IMAGE-arm64

bin/node/src/chain_spec/mod.rs

@@ -56,6 +56,10 @@ pub use node_runtime::GenesisConfig;
 
 type AccountPublic = <Signature as Verify>::Signer;
 
+pub fn joystream_mainnet_config() -> Result<ChainSpec, String> {
+    ChainSpec::from_json_bytes(&include_bytes!("../../../../joy-mainnet.json")[..])
+}
+
 /// Node `ChainSpec` extensions.
 ///
 /// Additional parameters for some Substrate core modules,

bin/node/src/command.rs

@@ -69,11 +69,11 @@ impl SubstrateCli for Cli {
     fn load_spec(&self, id: &str) -> std::result::Result<Box<dyn sc_service::ChainSpec>, String> {
         let spec = match id {
             "" => {
-                return Err(
-                    "Please specify which chain you want to run, e.g. --dev or --chain=local or --chain=prod-test"
-                        .into(),
-                )
+                return Err("Please specify which chain you want to run, eg. --dev or
+                    --chain=joy-mainnet or --chain=local or --chain=prod-test"
+                    .into())
             }
+            "joy-mainnet" => Box::new(chain_spec::joystream_mainnet_config()?),
             "dev" => Box::new(chain_spec::development_config()),
             "prod-test" => Box::new(chain_spec::prod_test_config()),
             "local" => Box::new(chain_spec::local_testnet_config()),

cli/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@joystream/cli",
   "description": "Command Line Interface for Joystream community and governance activities",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "author": "Leszek Wiesner",
   "bin": {
     "joystream-cli": "./bin/run"

cli/src/commands/content/channelPayoutProofAtByteOffset.ts

@@ -1,4 +1,5 @@
 import { channelPayoutProofAtByteOffset } from '@joystream/js/content'
+import { readBytesFromFile } from '@joystream/js/utils'
 import { Command, flags } from '@oclif/command'
 import { displayCollapsedRow } from '../../helpers/display'
 
@@ -36,8 +37,8 @@ export default class ChannelPayoutProofAtByteOffset extends Command {
       }
 
       const payoutProof = path
-        ? await channelPayoutProofAtByteOffset('PATH', path, start)
-        : await channelPayoutProofAtByteOffset('URL', url!, start)
+        ? await channelPayoutProofAtByteOffset(readBytesFromFile('PATH', path), start)
+        : await channelPayoutProofAtByteOffset(readBytesFromFile('URL', url!), start)
 
       displayCollapsedRow({
         'Channel Id': payoutProof.channelId,

cli/src/commands/content/channelPayoutsPayloadHeader.ts

@@ -1,5 +1,6 @@
 import { ChannelPayoutsMetadata } from '@joystream/metadata-protobuf'
 import { serializedPayloadHeader } from '@joystream/js/content'
+import { readBytesFromFile } from '@joystream/js/utils'
 import { Command, flags } from '@oclif/command'
 import chalk from 'chalk'
 import { displayCollapsedRow, displayTable } from '../../helpers/display'
@@ -27,8 +28,8 @@ export default class ChannelPayoutPayloadHeader extends Command {
 
     try {
       const serializedHeader = path
-        ? await serializedPayloadHeader('PATH', path)
-        : await serializedPayloadHeader('URL', url!)
+        ? await serializedPayloadHeader(readBytesFromFile('PATH', path))
+        : await serializedPayloadHeader(readBytesFromFile('URL', url!))
 
       const header = ChannelPayoutsMetadata.Header.decode(serializedHeader)
       this.log(

cli/src/commands/content/generateChannelPayoutsCommitment.ts

@@ -1,4 +1,5 @@
 import { generateCommitmentFromPayloadFile } from '@joystream/js/content'
+import { readBytesFromFile } from '@joystream/js/utils'
 import { flags } from '@oclif/command'
 import chalk from 'chalk'
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
@@ -27,8 +28,8 @@ export default class GenerateChannelPayoutsCommitment extends ContentDirectoryCo
 
     try {
       const commitment = path
-        ? await generateCommitmentFromPayloadFile('PATH', path)
-        : await generateCommitmentFromPayloadFile('URL', url!)
+        ? await generateCommitmentFromPayloadFile(readBytesFromFile('PATH', path))
+        : await generateCommitmentFromPayloadFile(readBytesFromFile('URL', url!))
 
       this.log(chalk.green(`Channel Payout payload merkle root is ${chalk.cyanBright(commitment)}!`))
     } catch (error) {

cli/src/commands/content/updateChannelPayoutsProposal.ts

@@ -4,6 +4,7 @@ import chalk from 'chalk'
 import UploadCommandBase from '../../base/UploadCommandBase'
 import fs from 'fs'
 import { createType } from '@joystream/types'
+import { readBytesFromFile } from '@joystream/js/utils'
 
 export default class UpdateChannelPayoutsProposal extends UploadCommandBase {
   static description = 'Create channel payouts proposal.'
@@ -94,7 +95,9 @@ export default class UpdateChannelPayoutsProposal extends UploadCommandBase {
       },
       {
         UpdateChannelPayouts: createType('PalletContentUpdateChannelPayoutsParametersRecord', {
-          commitment: payloadFilePath ? await generateCommitmentFromPayloadFile('PATH', payloadFilePath) : null,
+          commitment: payloadFilePath
+            ? await generateCommitmentFromPayloadFile(readBytesFromFile('PATH', payloadFilePath))
+            : null,
           payload: payloadFilePath
             ? {
                 objectCreationParams: {

distributor-node/CHANGELOG.md

@@ -1,4 +1,11 @@
-##
+## 1.5.0
+
+- Changed Elasticsearch transport to use data streams instead of regular indices. Renamed `config.logs.elastic.index` to `config.logs.elastic.indexPrefix`. Node ID from config will be automatically appended to the index name.
+
+## 1.4.1
+
+- Bumped `winston-elasticsearch` package verion
+- **FIX**: Added error handler to caught exception in `ElasticsearchTransport` and gracefully log them
 
 ### 1.4.0
 

distributor-node/config.yml

@@ -20,17 +20,17 @@ logs:
   #     username: username
   #     password: password
 limits:
-  storage: 100G
+  storage: 500G
   maxConcurrentStorageNodeDownloads: 100
   maxConcurrentOutboundConnections: 300
   outboundRequestsTimeoutMs: 5000
   pendingDownloadTimeoutSec: 3600
-  maxCachedItemSize: 1G
+  maxCachedItemSize: 20G
   queryNodeCacheTTL: 60
 intervals:
   saveCacheState: 60
   checkStorageNodeResponseTimes: 60
-  cacheCleanup: 7200 # every 2h, cache cleanup currently is very heavy on QN
+  cacheCleanup: 86400 # every 24h, cache cleanup currently is very heavy on QN
 publicApi:
   port: 3334
 operatorApi:

distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options-properties-indexprefix.md

@@ -0,0 +1,3 @@
+## indexPrefix Type
+
+`string`

distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options.md

@@ -4,12 +4,12 @@
 
 # elastic Properties
 
-| Property              | Type     | Required | Nullable       | Defined by                                                                                                                                                                                                                        |
-| :-------------------- | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [level](#level)       | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-level.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/level")                |
-| [endpoint](#endpoint) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-endpoint.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/endpoint") |
-| [index](#index)       | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-index.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/index")       |
-| [auth](#auth)         | `object` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-auth.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/auth")         |
+| Property                    | Type     | Required | Nullable       | Defined by                                                                                                                                                                                                                              |
+| :-------------------------- | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [level](#level)             | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-level.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/level")                      |
+| [endpoint](#endpoint)       | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-endpoint.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/endpoint")       |
+| [indexPrefix](#indexprefix) | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-indexprefix.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/indexPrefix") |
+| [auth](#auth)               | `object` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-auth.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/auth")               |
 
 ## level
 
@@ -65,21 +65,21 @@ Elastichsearch endpoint to push the logs to (for example: <http://localhost:9200
 
 **URI**: the string must be a URI, according to [RFC 3986](https://tools.ietf.org/html/rfc3986 "check the specification")
 
-## index
+## indexPrefix
 
-Elasticsearch index to push the logs to. If not provided, will fallback to "distributor-node"
+Elasticsearch data stream prefix to push the logs to. `-${config.id}` will be automatically appended. If not provided, will fallback to "logs-argus"
 
-`index`
+`indexPrefix`
 
 *   is optional
 
 *   Type: `string`
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-index.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/index")
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-indexprefix.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/indexPrefix")
 
-### index Type
+### indexPrefix Type
 
 `string`
 

distributor-node/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@joystream/distributor-cli",
   "description": "Joystream distributor node CLI",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "author": "Joystream contributors",
   "bin": {
     "joystream-distributor": "./bin/run"
@@ -50,7 +50,7 @@
     "url-join": "^4.0.1",
     "winston": "^3.3.3",
     "winston-daily-rotate-file": "^4.5.5",
-    "winston-elasticsearch": "^0.15.8",
+    "winston-elasticsearch": "^0.17.4",
     "yaml": "^1.10.2"
   },
   "devDependencies": {
@@ -161,7 +161,7 @@
     "checks": "tsc --noEmit --pretty && prettier ./ --check && yarn lint",
     "cli": "./bin/run",
     "start": "./bin/run start",
-    "start:with-instrumentation": "export OTEL_APPLICATION=distributor-node; node --require @joystream/opentelemetry ./bin/run start"
+    "start:with-instrumentation": "export OTEL_APPLICATION=distributor-node; export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8200; export OTEL_RESOURCE_ATTRIBUTES=service.name=argus,deployment.environment=production; node --require @joystream/opentelemetry ./bin/run start"
   },
   "types": "lib/index.d.ts"
 }

distributor-node/src/schemas/configSchema.ts

@@ -105,9 +105,10 @@ export const configSchema: JSONSchema4 = objectSchema({
               type: 'string',
               format: 'uri',
             },
-            index: {
+            indexPrefix: {
               description:
-                'Elasticsearch index to push the logs to. If not provided, will fallback to "distributor-node"',
+                // eslint-disable-next-line no-template-curly-in-string
+                'Elasticsearch data stream prefix to push the logs to. `-${config.id}` will be automatically appended. If not provided, will fallback to "logs-argus"',
               type: 'string',
             },
             auth: objectSchema({

distributor-node/src/services/logging/LoggingService.ts

@@ -70,19 +70,55 @@ export class LoggingService {
   private constructor(options: LoggerOptions, esTransport?: ElasticsearchTransport) {
     this.esTransport = esTransport
     this.rootLogger = winston.createLogger(options)
+
+    // Compulsory error handling
+    this.rootLogger.on('error', (error) => {
+      console.error('Error in logger caught:', error)
+    })
+    this.esTransport?.on('error', (error) => {
+      console.error('Error in logger caught:', error)
+    })
   }
 
   public static withAppConfig(config: ReadonlyConfig): LoggingService {
     const transports: winston.LoggerOptions['transports'] = []
 
     let esTransport: ElasticsearchTransport | undefined
     if (config.logs?.elastic) {
+      const indexPrefix = config.logs.elastic.indexPrefix || 'logs-argus'
+      const index = `${indexPrefix}-${config.id}`.toLowerCase()
       esTransport = new ElasticsearchTransport({
-        index: config.logs.elastic.index || 'distributor-node',
+        index,
+        dataStream: true,
         level: config.logs.elastic.level,
         format: winston.format.combine(pauseFormat({ id: 'es' }), escFormat()),
         retryLimit: 10,
         flushInterval: 5000,
+        // apply custom transform so that tracing data (if present) is placed in the top level of the log
+        // based on https://github.com/vanthome/winston-elasticsearch/blob/d948fa1b705269a4713480593ea657de34c0a942/transformer.js
+        transformer: (logData) => {
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          const transformed: any = {}
+          transformed['@timestamp'] = logData.timestamp ? logData.timestamp : new Date().toISOString()
+          transformed.message = logData.message
+          transformed.severity = logData.level
+          transformed.fields = logData.meta
+
+          if (logData.meta.trace_id || logData.meta.trace_flags) {
+            transformed.trace = {
+              id: logData.meta.trace_id,
+              flags: logData.meta.trace_flags,
+            }
+          }
+          if (logData.meta.span_id) {
+            transformed.span = { id: logData.meta.span_id }
+          }
+          if (logData.meta.transaction_id) {
+            transformed.transaction = { id: logData.meta.transaction_id }
+          }
+
+          return transformed
+        },
         source: config.id,
         clientOpts: {
           node: {

docker-compose.elasticsearch.yml

@@ -19,7 +19,7 @@ services:
     volumes:
       - es-data:/usr/share/elasticsearch/data
     ports:
-      - 127.0.0.1:9200:9200
+      - 9200:9200
     networks:
       - joystream_default
 

joystream-node.Dockerfile

@@ -21,6 +21,7 @@ COPY Cargo.lock .
 COPY bin ./bin
 COPY runtime ./runtime
 COPY runtime-modules ./runtime-modules
+COPY joy-mainnet.json .
 ARG CARGO_UNSTABLE_SPARSE_REGISTRY=true
 ARG CARGO_NET_GIT_FETCH_WITH_CLI=true
 RUN cargo +nightly chef prepare --recipe-path /joystream/recipe.json
@@ -43,6 +44,7 @@ COPY Cargo.lock .
 COPY bin ./bin
 COPY runtime ./runtime
 COPY runtime-modules ./runtime-modules
+COPY joy-mainnet.json .
 # Copy over the cached dependencies
 COPY --from=cacher /joystream/target target
 COPY --from=cacher $CARGO_HOME $CARGO_HOME

joystreamjs/CHANGELOG.md

@@ -1,3 +1,7 @@
+### 1.6.0
+
+- rework `util/readBytesFromFile` to support use in browser [#4584](https://github.com/Joystream/joystream/pull/4584)
+
 ### 1.5.0
 
 - Update dependency on metadata protobuf to v2.10.0