Merge pull request #4970 from mnaamani/testing-script-cleanup

Test script runners refactoring

.github/workflows/joystream-apps-docker.yml

@@ -87,7 +87,7 @@ jobs:
       - name: Test with latest images
         if: github.event_name == 'pull_request'
         run: |
-          yarn build:packages
+          yarn build
           cp docker-compose-no-bind-volumes.yml docker-compose.yml
           ./tests/network-tests/run-tests.sh content-directory
 

.github/workflows/lint-typescript.yml

@@ -16,7 +16,7 @@ jobs:
         node-version: ${{ matrix.node-version }}
     - name: lint
       run: |
-        yarn build:packages
+        yarn build
         yarn lint
 
   lint_osx:
@@ -33,5 +33,5 @@ jobs:
         node-version: ${{ matrix.node-version }}
     - name: lint
       run: |
-        yarn build:packages
+        yarn build
         yarn lint

.github/workflows/run-network-tests.yml

@@ -21,7 +21,7 @@ jobs:
         node-version: ${{ matrix.node-version }}
     - name: checks
       run: |
-        yarn build:packages
+        yarn build
 
   tests_build_osx:
     name: MacOS Checks
@@ -37,7 +37,7 @@ jobs:
         node-version: ${{ matrix.node-version }}
     - name: checks
       run: |
-        yarn build:packages
+        yarn build
 
   build_images:
     name: Build joystream/node
@@ -166,7 +166,7 @@ jobs:
           docker images
       - name: Install packages and dependencies
         run: |
-          yarn build:packages
+          yarn build
       - name: Ensure tests are runnable
         run: yarn workspace network-tests build
       - name: Execute network tests
@@ -231,7 +231,7 @@ jobs:
       - name: Install packages and dependencies
         if: env.GIT_DIFF
         run: |
-          yarn build:packages
+          yarn build
       - name: Ensure tests are runnable
         if: env.GIT_DIFF
         run: yarn workspace network-tests build

README.md

@@ -51,13 +51,13 @@ Modify the root `package.json` and change volta section to use node version 16.2
 
 ```sh
 # Build local npm packages
-yarn build:packages
+yarn build
 
 # Build joystream/node docker testing image
 RUNTIME_PROFILE=TESTING yarn build:node:docker
 
 # Start a local development network
-RUNTIME_PROFILE=TESTING yarn start
+yarn start
 ```
 
 ## Software
@@ -111,13 +111,13 @@ Look under the 'Assets' section:
 
 ```bash
 # Make sure yarn packages are built
-yarn build:packages
+yarn build
 
 # Build the test joystream-node
 RUNTIME_PROFILE=TESTING yarn build:node:docker
 
 # Run tests
-./tests/network-tests/run-full-tests.sh
+yarn test
 ```
 
 ### Contributing

RUNTIME-CONTRIBUTING.md

@@ -225,7 +225,7 @@ be followed below:
   1. Re run code formatting with `cargo fmt --all`
   1. Do another test build with modified benchmarks `yarn cargo-checks && yarn cargo-build`
   1. Extract the new runtime metadata `yarn update-chain-metadata`
-  1. Build all npm packages: `yarn build:packages`
+  1. Build all npm packages: `yarn build`
   1. Add any new integration tests, query-node mappings that cover the changes implemented.
   1. Lint typescript `yarn lint`
   1. Build the testing runtime joystream/node docker image and run the full integration test suite (see Integration tests section below)

devops/ansible/roles/common/tasks/build-apps.yml

@@ -4,7 +4,7 @@
 - name: Build joystream apps
   shell: 
     cmd: |
-      bash -ic "yarn build:packages"
+      bash -ic "yarn build"
   args:
     chdir: '{{ remote_code_path }}'
     # Always try to rebuild incase code has been updated for existing repo

devops/extrinsic-ordering/filter.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# This script is used in a Github Workflow. It helps filtering out what is interesting
+# when comparing metadata and spot what would require a tx version bump.
+
+# shellcheck disable=SC2002,SC2086
+
+FILE=$1
+
+# Higlight indexes that were deleted
+function find_deletions() {
+    echo "\n## Deletions\n"
+    RES=$(cat "$FILE" | grep -n '\[\-\]' | tr -s " ")
+    if [ "$RES" ]; then
+        echo "$RES" | awk '{ printf "%s\\n", $0 }'
+    else
+        echo "n/a"
+    fi
+}
+
+# Highlight indexes that have been deleted
+function find_index_changes() {
+    echo "\n## Index changes\n"
+    RES=$(cat "$FILE" | grep -E -n -i 'idx:\s*([0-9]+)\s*(->)\s*([0-9]+)' | tr -s " ")
+    if [ "$RES" ]; then
+        echo "$RES" | awk '{ printf "%s\\n", $0 }'
+    else
+        echo "n/a"
+    fi
+}
+
+# Highlight values that decreased
+function find_decreases() {
+    echo "\n## Decreases\n"
+    OUT=$(cat "$FILE" | grep -E -i -o '([0-9]+)\s*(->)\s*([0-9]+)' | awk '$1 > $3 { printf "%s;", $0 }')
+    IFS=$';' LIST=("$OUT")
+    unset RES
+    for line in "${LIST[@]}"; do
+        RES="$RES\n$(cat "$FILE" | grep -E -i -n \"$line\" | tr -s " ")"
+    done
+
+    if [ "$RES" ]; then
+        echo "$RES" | awk '{ printf "%s\\n", $0 }' | sort -u -g | uniq
+    else
+        echo "n/a"
+    fi
+}
+
+echo "\n------------------------------ SUMMARY -------------------------------"
+echo "\n⚠️ This filter is here to help spotting changes that should be reviewed carefully."
+echo "\n⚠️ It catches only index changes, deletions and value decreases".
+
+find_deletions "$FILE"
+find_index_changes "$FILE"
+find_decreases "$FILE"
+echo "\n----------------------------------------------------------------------\n"

devops/extrinsic-ordering/tx-ordering.yml

@@ -0,0 +1,81 @@
+# This workflow performs the Extrinsic Ordering Check on demand using a binary
+
+name: Release - Extrinsic Ordering Check
+on:
+  workflow_dispatch:
+    inputs:
+      reference_url:
+        description: The WebSocket url of the reference node
+        default: wss://kusama-rpc.polkadot.io
+        required: true
+      binary_url:
+        description: A url to a Linux binary for the node containing the runtime to test
+        default: https://releases.parity.io/polkadot/x86_64-debian:stretch/v0.9.10/polkadot
+        required: true
+      chain:
+        description: The name of the chain under test. Usually, you would pass a local chain
+        default: kusama-local
+        required: true
+
+jobs:
+  check:
+    name: Run check
+    runs-on: ubuntu-latest
+    env:
+      CHAIN: ${{github.event.inputs.chain}}
+      BIN_URL: ${{github.event.inputs.binary_url}}
+      REF_URL: ${{github.event.inputs.reference_url}}
+
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Fetch binary
+        run: |
+          echo Fetching $BIN_URL
+          wget $BIN_URL
+          chmod a+x polkadot
+          ./polkadot --version
+
+      - name: Start local node
+        run: |
+          echo Running on $CHAIN
+          ./polkadot --chain=$CHAIN &
+
+      - name: Prepare output
+        run: |
+          VERSION=$(./polkadot --version)
+          echo "Metadata comparison:" >> output.txt
+          echo "Date: $(date)" >> output.txt
+          echo "Reference: $REF_URL" >> output.txt
+          echo "Target version: $VERSION" >> output.txt
+          echo "Chain: $CHAIN" >> output.txt
+          echo "----------------------------------------------------------------------" >> output.txt
+
+      - name: Pull polkadot-js-tools image
+        run: docker pull jacogr/polkadot-js-tools
+
+      - name: Compare the metadata
+        run: |
+          CMD="docker run --pull always --network host jacogr/polkadot-js-tools metadata $REF_URL ws://localhost:9944"
+          echo -e "Running:\n$CMD"
+          $CMD >> output.txt
+          sed -z -i 's/\n\n/\n/g' output.txt
+          cat output.txt | egrep -n -i ''
+          SUMMARY=$(./scripts/ci/github/extrinsic-ordering-filter.sh output.txt)
+          echo -e $SUMMARY
+          echo -e $SUMMARY >> output.txt
+
+      - name: Show result
+        run: |
+          cat output.txt
+
+      - name: Stop our local node
+        run: pkill polkadot
+
+      - name: Save output as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.CHAIN }}
+          path: |
+            output.txt

package.json

@@ -6,8 +6,10 @@
   "scripts": {
     "build:node:docker": "./build-node-docker.sh",
     "build:packages": "./build-packages.sh",
+    "build": "./build-packages.sh",
     "setup": "./setup.sh",
     "start": "./start.sh",
+    "test": "./tests/network-tests/run-tests.sh",
     "cargo-checks": "./scripts/cargo-checks.sh",
     "cargo-build": "./scripts/cargo-build.sh",
     "lint": "./scripts/lint-typescript.sh",

query-node/README.md

@@ -10,7 +10,7 @@ Run
 
 ```bash
 # Make sure these steps are executed first:
-# yarn build:packages; # OR:
+# yarn build; # OR:
 # yarn && yarn workspace @joystream/types build && yarn workspace @joystream/metadata-protobuf build
 yarn workspace query-node-root build
 ```
@@ -70,13 +70,13 @@ GRAPHQL_PLAYGROUND_CDN="query/server" yarn workspace query-node-root query-node:
 Run integration tests
 
 ```
-./query-node/run-tests.sh
+./tests/network-tests/run-tests.sh
 ```
 
-To run tests and keep services alive for further inspection, set `DEBUG` shell variable to any true-ish value.
+To run tests and keep services alive for further inspection, set `PERSIST` shell variable to any true-ish value.
 
 ```
-DEBUG=true ./query-node/run-tests.sh
+PERSIST=true ./tests/network-tests/run-tests.sh
 ```
 
 You can then use queries manually in GraphQL Playground (http://localhost:8081/graphql),
@@ -87,8 +87,9 @@ This assumes the scenario is repeatable and any previous test errors didn't brea
 the blockchain or processor state in a critical way.
 
 ```
-DEBUG=true ./query-node/run-tests.sh # run tests first and make sure services stay alive
-REUSE_KEYS=true yarn workspace network-tests run-test-scenario content-directory
+# run tests first and make sure services stay alive
+PERSIST=true ./tests/network-tests/run-tests.sh
+yarn workspace network-tests run-test-scenario content-directory
 ```
 
 Commenting out some of the scenario's flow calls in `network-tests/src/scenarios/content-directory.ts` is not relevant to the current

query-node/start.sh

@@ -19,6 +19,10 @@ docker-compose -f ../docker-compose.yml up -d hydra-indexer-gateway
 # Start processor
 docker-compose -f ../docker-compose.yml up -d processor
 echo "Waiting for processor to be ready..." && sleep 30
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    # On Docker Desktop things take a bit longer to startup
+    sleep 150
+fi
 
 # Start graphql-server
 docker-compose -f ../docker-compose.yml up -d graphql-server

start-multistorage.sh

@@ -1,7 +1,13 @@
 #!/usr/bin/env bash
 set -e
 
+SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
+cd $SCRIPT_PATH
+
+rm tests/network-tests/output.json || :
+
 # Run a complete joystream development network on your machine using docker
+export RUNTIME_PROFILE=${RUNTIME_PROFILE:=TESTING}
 export JOYSTREAM_NODE_TAG=${JOYSTREAM_NODE_TAG:=$(./scripts/runtime-code-shasum.sh)}
 
 INIT_CHAIN_SCENARIO=${INIT_CHAIN_SCENARIO:=setupNewChainMultiStorage}

start.sh

@@ -1,7 +1,13 @@
 #!/usr/bin/env bash
 set -e
 
+SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
+cd $SCRIPT_PATH
+
+rm tests/network-tests/output.json || :
+
 # Run a complete joystream development network on your machine using docker
+export RUNTIME_PROFILE=${RUNTIME_PROFILE:=TESTING}
 export JOYSTREAM_NODE_TAG=${JOYSTREAM_NODE_TAG:=$(./scripts/runtime-code-shasum.sh)}
 
 INIT_CHAIN_SCENARIO=${INIT_CHAIN_SCENARIO:=setupNewChain}