From 932f2b23691d17f1a044d2a42fccf070444085df Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 29 Jul 2024 17:27:17 -0700 Subject: [PATCH] Olshansk debugging round2 (#10) * Debugging from discord * Added clean commands * Running locally, but still trying to figure out usage * Instructions on using postgres * Add amd64 and arm64 to build targets on CI to ensure both are properly working. * Added @Olshansk change on dev-node.dockerfile (dev & test) to node.dockerfile (production) --------- Co-authored-by: Jorge S. Cuesta --- .github/workflows/ci-docker.yml | 4 + .gitignore | 4 + README.md | 55 +++++++++--- docker/dev-node.dockerfile | 7 +- docker/node.dockerfile | 6 +- docs/introduction.md | 145 ++++++++++++++++++++------------ scripts/prepare-dotenv.sh | 5 +- 7 files changed, 157 insertions(+), 69 deletions(-) diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index 4ae0daf..00ab6f6 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -29,6 +29,8 @@ jobs: file: docker/node.dockerfile cache-from: type=gha cache-to: type=gha,mode=max + # Added platforms to ensure this image will work on Linux and OSx with Mx processors + platforms: linux/amd64,linux/arm64 build-args: | CI=true NODE_ENV=production @@ -42,6 +44,8 @@ jobs: file: docker/dev-node.dockerfile cache-from: type=gha cache-to: type=gha,mode=max + # Added platforms to ensure this image will work on Linux and OSx with Mx processors + platforms: linux/amd64,linux/arm64 build-args: | CI=true NODE_ENV=test diff --git a/.gitignore b/.gitignore index 9b1b933..69600df 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,7 @@ Thumbs.db !.env.sample .yarn/cache + +# Visual Studio +.vscode + diff --git a/README.md b/README.md index 6eeace7..fa69fa8 100644 --- a/README.md +++ b/README.md @@ -14,15 +14,18 @@ API to the indexed data. To learn more about SubQuery, [see their docs](https://academy.subquery.network). -- [Docs](#docs) +- [Usage \& Query Docs](#usage--query-docs) + - [Explore via postgres](#explore-via-postgres) - [Getting Started](#getting-started) + - [tl;dr local development (if not your first time)](#tldr-local-development-if-not-your-first-time) - [1. Ensure submodules are updated](#1-ensure-submodules-are-updated) - [2. Install dependencies](#2-install-dependencies) - [3. Generate types](#3-generate-types) - [4. Run](#4-run) - - [4.1 Errors running \& building](#41-errors-running--building) + - [Localnet ONLY](#localnet-only) + - [4.1 Debugging, errors running \& building](#41-debugging-errors-running--building) - [4.2 Using a pre-built image](#42-using-a-pre-built-image) - - [4.3 Available scripts breakdown](#43-available-scripts-breakdown) + - [4.3 Available Scripts breakdown](#43-available-scripts-breakdown) - [DB Migrations](#db-migrations) - [Install dependencies](#install-dependencies) - [Running Migrations](#running-migrations) @@ -36,12 +39,36 @@ To learn more about SubQuery, [see their docs](https://academy.subquery.network) - [End-to-end Testing](#end-to-end-testing) - [Tracing](#tracing) -## Docs +## Usage & Query Docs -See the [docs](./docs/introduction.md) directory. +See the [introduction docs](./docs/introduction.md) directory for details +on how to use indexed data after you're fully set up. + +### Explore via postgres + +Connect to the postgres container, update the schema and explore! + +```bash +docker exec -it pocketdex_development-postgres-1 psql -U postgres -d postgres +SET search_path TO app; +\dt +``` ## Getting Started +### tl;dr local development (if not your first time) + +Run the following: + +```bash +yarn install +yarn run codegen +docker context use default # Optional +yarn run docker:build:development +yarn poktroll:proxy:start +yarn run docker:start:development +``` + ### 1. Ensure submodules are updated ```shell @@ -68,11 +95,13 @@ Dotenv files will be automatically created after the `yarn install` thanks to th After that, feel free to modify them as you wish. You will see three dotenv files, each for the corresponding script and environment: + * `.env.production` * `.env.development` * `.env.test` Alternatively, you can manually create them running: + ```shell yarn run env:prepare ``` @@ -80,7 +109,8 @@ yarn run env:prepare For this README we will be running all the commands in `development` but you can also run them in `test` or `production`. Following this structure, you can run every docker command `docker::`, -#### Localnet ONLY: +#### Localnet ONLY + ```shell # Run this ONLY IF indexing poktroll localnet. # This will allows subquery-node to connect with the poktroll validator @@ -97,7 +127,7 @@ Build & start: ```shell # Then build docker and start yarn run docker:build:development -# This will turn on the process under a WATCHER so any change to the project.ts schema.graphql or src will trigger +# This will turn on the process under a WATCHER so any change to the project.ts schema.graphql or src will trigger # the needed builds again. yarn run docker:start:development ``` @@ -114,15 +144,18 @@ Or Stop & clean up (delete postgres data): yarn run docker:clean:development ``` -#### 4.1 Errors running & building +#### 4.1 Debugging, errors running & building -If you're hitting errors with the above command, try cleaning your cache: +If you're hitting errors with the above command, do a nuclear clean of all potential issues: ```bash yarn cache clean +yarn vendor:clean +docker builder prune --all +docker context use default ``` -And pick up from the `yarn run docker:build` step above +Now pick up from the `yarn run docker:build` step above. #### 4.2 Using a pre-built image @@ -175,7 +208,7 @@ services: * `docker:start:` - Starts all services for the specified environment. * `docker:ps:` - Shows the status of services for the specified environment. * `docker:stop:` - Stops all active services for the specified environment without removing them. -* `docker:clean:` - Stops and removes all services, volumes, and networks for the specified environment. +* `docker:clean:` - Stops and removes all services, volumes, and networks for the specified environment. ## DB Migrations diff --git a/docker/dev-node.dockerfile b/docker/dev-node.dockerfile index f77ad31..785e7ae 100644 --- a/docker/dev-node.dockerfile +++ b/docker/dev-node.dockerfile @@ -12,7 +12,12 @@ ENV CI=$CI # typescript is added here because is wrongly used on some of the workspaces, just by the name # without the use of npm exec, yarn exec or any other to ensure they are looking into the node_modules -RUN apt-get update && apt-get install -y tree git postgresql-client tini curl jq yq && npm i -g typescript +RUN apt-get update \ + && apt-get install -y tree git postgresql-client tini curl jq yq \ + # The following was necessary to install in order to add support for building + # the docker container on an M1 chip (i.e. ARM64) + make build-essential pkg-config python3 libusb-1.0-0-dev libudev-dev \ + && npm i -g typescript WORKDIR /app diff --git a/docker/node.dockerfile b/docker/node.dockerfile index 983f6df..ea43e70 100644 --- a/docker/node.dockerfile +++ b/docker/node.dockerfile @@ -11,7 +11,11 @@ ENV CHAIN_ID=$CHAIN_ID # Typescript is added here because is wrongly used on some of the workspaces, just by the name # without the use of npm exec, yarn exec or any other to ensure they are looking into the node_modules -RUN apt-get update && apt-get install -y tree && npm i -g typescript +RUN apt-get update && apt-get install -y tree \ + # The following was necessary to install in order to add support for building + # the docker container on an M1 chip (i.e. ARM64) + make build-essential pkg-config python3 libusb-1.0-0-dev libudev-dev \ + && npm i -g typescript WORKDIR /app diff --git a/docs/introduction.md b/docs/introduction.md index 6336342..3656c86 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -1,32 +1,63 @@ -# Introduction +# Query & Usage Introduction + +- [Introduction](#introduction) +- [\[TODO\] Endpoints / Playground UIs](#todo-endpoints--playground-uis) +- [Architecture](#architecture) + - [Component Diagram](#component-diagram) +- [Querying](#querying) + - [Pagination](#pagination) + - [Filtering](#filtering) + - [Filtering Examples](#filtering-examples) + - [Order by / Sorting](#order-by--sorting) + - [Block height](#block-height) + - [Contract Code ID](#contract-code-id) + - [Order direction](#order-direction) + - [Aggregation](#aggregation) + - [Tests as examples](#tests-as-examples) +- [Entities](#entities) + - [Primitive entities](#primitive-entities) + - [Entity relationship diagrams](#entity-relationship-diagrams) +- [Versioning](#versioning) + - [Versioning Example](#versioning-example) + - [`"_metadata"` Entity](#_metadata-entity) + - [Metadata Query Example](#metadata-query-example) + +## Introduction + +Pocketdex is a [SubQuery](https://www.subquery.network/)-based indexer for the +Shannon implementation of the pocket network protocol distributed ledger. -Pocketdex is a [SubQuery](https://www.subquery.network/)-based indexer for the Shannon implementation of the pocket network protocol distributed ledger. This indexer provides a [Graphql](https://www.subquery.network/) API for querying tracked entities. For a list of tracked entities, see the [schema.graphql file](../schema.graphql). -While the SubQuery SDK supports adding arbitrary entities, in order to support forward compatibility, this indexer tracks all primitives such that they can be used to construct higher-level entities in future migrations. +While the SubQuery SDK supports adding arbitrary entities, in order to support +forward compatibility, this indexer tracks all primitives such that they can be +used to construct higher-level entities in future migrations. + See [primitive entities](#primitive-entities) for more information. -To learn more about how to [run](https://academy.subquery.network/run_publish/run.html) or [change this SubQuery Project](https://academy.subquery.network/quickstart/quickstart_chains/cosmos.html) to get your own custom GraphQL API for your app, [visit the SubQuery Academy for documentation](https://academy.subquery.network/). +To learn more about how to [run](https://academy.subquery.network/run_publish/run.html) +or [change this SubQuery Project](https://academy.subquery.network/quickstart/quickstart_chains/cosmos.html) +to get your own custom GraphQL API for your app, [visit the SubQuery Academy for documentation](https://academy.subquery.network/). -## Endpoints / Playground UIs +## [TODO] Endpoints / Playground UIs The graphql API endpoints also serve a playground UI to browsers for convenience. This UI is useful for rapid experimentation and iteration of queries as well as just getting some results, features include: - real-time query results - query editor: - - auto-complete & validation via schema introspection - - can store multiple, named queries - - supports graphql variables + - auto-complete & validation via schema introspection + - can store multiple, named queries + - supports graphql variables - local persistence of query editor contents - schema reference - graphql docs reference -| Network | API / Playground URL | -| --- | --- | +_TODO: Fll this out_ -_TODO_ +| Network | API / Playground URL | +| ------- | -------------------- | ## Architecture @@ -62,7 +93,7 @@ pg[Postgres DB] pgr -.-> pg subgraph sqn[SubQuery Node] - + fs[Fetch Service] ss[Store Service] im[Indexer Manager] @@ -77,11 +108,13 @@ fs -.-> pv ``` ## Querying + The graphql API relies heavily on [postgraphile (as a library)](https://www.graphile.org/postgraphile/usage-library/) to resolve graphql requests. Postgraphile plugins also play a critical role; in particular, the [connection-filter](https://github.com/graphile-contrib/postgraphile-plugin-connection-filter) and [pg-aggregates](https://github.com/graphile/pg-aggregates) plugins. ### Pagination + The graphql API implements [the connections specification](https://relay.dev/graphql/connections.htm) for pagination (see: [GraphQL pagination docs](https://graphql.org/learn/pagination/#end-of-list-counts-and-connections) for more). **It is recommended to prefer using pagination operators by default (e.g. `first: `) to avoid unnecessary delays in query responses.** @@ -91,20 +124,16 @@ The graphql API implements [the connections specification](https://relay.dev/gra Filtering is facilitated by postgraphile and its plugins. For specifics on supported operators and how to use them, please refer to their documentation: - [connection-filter plugin](https://github.com/graphile-contrib/postgraphile-plugin-connection-filter) - - [operators](https://github.com/graphile-contrib/postgraphile-plugin-connection-filter/blob/master/docs/operators.md#json-jsonb) - - [query examples](https://github.com/graphile-contrib/postgraphile-plugin-connection-filter/blob/master/docs/examples.md) + - [operators](https://github.com/graphile-contrib/postgraphile-plugin-connection-filter/blob/master/docs/operators.md#json-jsonb) + - [query examples](https://github.com/graphile-contrib/postgraphile-plugin-connection-filter/blob/master/docs/examples.md) -#### Examples +#### Filtering Examples Filtering `NativeTransfer`s for a given sender address: ```graphql query nativeTransfersFromAddress { - nativeTransfers(first: 5, filter: { - fromAddress: { - equalTo: "fetch1t3qet68dr0qkmrjtq89lrx837qa2t05265qy6s" - } - }) { + nativeTransfers(first: 5, filter: {fromAddress: {equalTo: "fetch1t3qet68dr0qkmrjtq89lrx837qa2t05265qy6s"}}) { nodes { toAddress amounts @@ -117,13 +146,10 @@ Filtering for `Message`s from a given sender address: ```graphql query messagesFromAddress { - messages (first: 5, filter: { - transaction: { - signerAddress: { - equalTo: "fetch1t3qet68dr0qkmrjtq89lrx837qa2t05265qy6s" - } - } - }) { + messages( + first: 5 + filter: {transaction: {signerAddress: {equalTo: "fetch1t3qet68dr0qkmrjtq89lrx837qa2t05265qy6s"}}} + ) { nodes { transaction { signerAddress @@ -137,15 +163,15 @@ Filtering on `Events`s within a given timeframe and with a given type: ```graphql query transferEventsDuring { - events(first: 5, filter: { - block: { - timestamp: { - greaterThanOrEqualTo: "2022-09-15T01:44:13.719", - lessThanOrEqualTo: "2022-09-19T02:15:28.632" + events( + first: 5 + filter: { + block: { + timestamp: {greaterThanOrEqualTo: "2022-09-15T01:44:13.719", lessThanOrEqualTo: "2022-09-19T02:15:28.632"} } - }, - type: {equalTo: "transfer"}, - }) { + type: {equalTo: "transfer"} + } + ) { nodes { attributes { nodes { @@ -159,11 +185,14 @@ query transferEventsDuring { ``` ### Order by / Sorting + Each entity, by default, can be sorted by any of its respective fields. Additional support for ordering by certain fields on related entities is facilitated by custom ordering plugins generated from `makeAddPgTableOrderByPlugin` (see: [postgraphile-docs](https://www.graphile.org/postgraphile/make-add-pg-table-order-by-plugin/)). #### Block height + Any entity which relates to `Block` can be ordered by a related block's `height` field: + ```graphql query contractExecByBlockHeight { contractExecutionMessage (orderBy: EXECUTE_CONTRACT_MESSAGES_BY_BLOCK_HEIGHT_ASC) { @@ -179,7 +208,9 @@ query contractExecByBlockHeight { ``` #### Contract Code ID + The `contract` entity can be sorted by `codeId` through the `storeMessage` and `instantiateMessage` relations. + ```graphql query contractsByRelatedCodeID { contracts (orderBy: CONTRACTS_BY_STORE_CONTRACT_MESSAGES_CODE_ID_ASC) { @@ -196,6 +227,7 @@ query contractsByRelatedCodeID { ``` #### Order direction + Each of these custom orders are implemented in both directions, ascending and descending. These directions are accessed through the ending characters of the order enum, by choosing either `_ASC` and `_DESC`. ### Aggregation @@ -310,9 +342,10 @@ Event ||--|{ EventAttribute : "" ``` ## Versioning + The versions of both the GraphQL API and the Indexer itself can be retrieved simply using the following query on the GraphQL playground. -##### Example: +### Versioning Example ```graphql query ReleaseVersionTest { @@ -324,34 +357,40 @@ query ReleaseVersionTest { ``` Each of these version numbers are stored as the value to the key `"version"` within their relevant module `package.json` file. These files can be found in the `docker/node-cosmos/` and `subql/packages/query/` directories for the Indexer and GraphQL versions, respectively. + ```yaml // The Indexer version number, taken from "docker/node-cosmos/package.json" -{ +{ "name": "@subql/node-cosmos", "version": "1.0.0", ... } ``` -#### `"_metadata"` Entity +### `"_metadata"` Entity + The `_metadata` entity has further utility beyond the scope of the example query given prior. Using any of the relevant fields from the type definition below, internal states and config information can be retrieved with ease. -``` + +```graphql type _Metadata { - lastProcessedHeight: Int - lastProcessedTimestamp: Date - targetHeight: Int - chain: String - specName: String - genesisHash: String - indexerHealthy: Boolean - indexerNodeVersion: String - queryNodeVersion: String - rowCountEstimate: [TableEstimate] - dynamicDatasources: String - } + lastProcessedHeight: Int + lastProcessedTimestamp: Date + targetHeight: Int + chain: String + specName: String + genesisHash: String + indexerHealthy: Boolean + indexerNodeVersion: String + queryNodeVersion: String + rowCountEstimate: [TableEstimate] + dynamicDatasources: String +} ``` -##### Example: + +#### Metadata Query Example + If a developer was curious about the `chain-id` or whether the Indexer has passed any health checks, using `indexerHealthy`, these values can be returned within the playground or otherwise connected projects. + ```graphql query ReleaseVersionTest { _metadata { @@ -359,4 +398,4 @@ query ReleaseVersionTest { indexerHealthy } } -``` \ No newline at end of file +``` diff --git a/scripts/prepare-dotenv.sh b/scripts/prepare-dotenv.sh index 0f00dc8..3bd7c1d 100755 --- a/scripts/prepare-dotenv.sh +++ b/scripts/prepare-dotenv.sh @@ -6,13 +6,12 @@ set -e # Check if .env.sample exists if [ ! -f .env.sample ]; then - warning_log ".env.sample does not exist. Skipping automatic dotenv files creation" + warning_log ".env.sample does not exist. Skipping automatic dotenv files creation" else # Define environments environments="development production test" - for environment in $environments - do + for environment in $environments; do # Create a copy of .env.sample for each environment cp .env.sample .env.$environment