Merge pull request #229 from kodadot/main

(RELEASE) Speck v11

.env

@@ -3,3 +3,4 @@ DB_PORT=23798
 PROCESSOR_PROMETHEUS_PORT=3000
 GQL_PORT=4350
 SQD_DEBUG=squid:log
+ALL_SQD_DEBUG=*

.eslintrc

@@ -1,6 +1,6 @@
 {
   "extends": ["eslint-config-unjs"],
-  "ignorePatterns": ["tests/**", "src/model", "lib/**"],
+  "ignorePatterns": ["tests/**", "src/model", "src/server-extension", "src/types", "lib/**"],
   "rules": {
     "@typescript-eslint/no-inferrable-types": 0,
     "@typescript-eslint/no-unused-vars": 0,
@@ -23,6 +23,7 @@
     "unicorn/prefer-code-point": 0,
     "unicorn/prefer-math-trunc": 0,
     "unicorn/text-encoding-identifier-case": 0,
-    "no-useless-constructor": 0
+    "no-useless-constructor": 0,
+    "semi": ["error", "never"]
   }
 }

.github/workflows/subsquid.yml

@@ -23,4 +23,4 @@ jobs:
           sqd auth -k ${{ secrets.DEPLOYMENT_KEY }}
       - name: deploy subsquid
         id: deploy
-        run: sqd deploy -u -m ${{ env.VERSION }}.yaml .
+        run: sqd deploy -o kodadot -u -m ${{ env.VERSION }}.yaml .

README.md

@@ -2,85 +2,90 @@
 
 ![](https://media.tenor.com/eK1dyB3TOLsAAAAC/anime-stick.gif)
 
-KodaDot's [Squid](https://docs.subsquid.io) based data processor for [KodaDot](https://kodadot.xyz) NFT Marketplace.
+[Squid](https://docs.subsquid.io) based data used to index, process, and query on top of AssetHub for [KodaDot](https://kodadot.xyz) NFT Marketplace.
+
+## Hosted Squids
+
+* Kusama AssetHub Processor (Statemine -> KSM): https://squid.subsquid.io/stick/graphql
+* Polkadot AssetHub Processor (Statemint -> DOT): https://squid.subsquid.io/speck/graphql
+* Rococo Testnet Processor: 🚧 WIP 🚧
+
+## Project structure
+
+* `src/generated` - model/server definitions created by `codegen`. Do not alter the contents of this directory manually.
+* `src/server-extension` - module with custom `type-graphql` based resolvers.
+* `src/types` - data type definitions for chain events and extrinsics created by `typegen`.
+* `src/mappings` - mapping module.
+* `lib` - compiled js files. The structure of this directory must reflect `src`.
+* `.env` - environment variables defined here or supplied by a shell.
 
 ## Prerequisites
 
-* node 18.x
-* docker
-* npm -- note that `yarn` package manager is not supported
+* Node 18.x
+* Docker
+* npm
+* [just](https://github.com/casey/just)
 
 ## Quickly running the sample
 
-Example commands below use [sqd](https://docs.subsquid.io/squid-cli/).
-Please [install](https://docs.subsquid.io/squid-cli/installation/) it before proceeding.
-
 ```bash
-# 1. Update Squid SDK and install dependencies
-npm run update
+# 1. Install dependencies
 npm ci
 
-# 2. Start target Postgres database and detach
-sqd up
+# 2. Build project
+just build
 
-# 3. Build the project and start the processor
-sqd process
+# 3. Start target Postgres database container
+just upd
 
-# 4. The command above will block the terminal
-#    being busy with fetching the chain data, 
-#    transforming and storing it in the target database.
-#
-#    To start the graphql server open a separate terminal
-#    and run
-sqd serve
-```
+# 4. Update database with data objects
+just migrate
 
-## Project structure
+# 5. Start the processor
+just process
 
-* `src/generated` - model/server definitions created by `codegen`. Do not alter the contents of this directory manually.
-* `src/server-extension` - module with custom `type-graphql` based resolvers
-* `src/types` - data type definitions for chain events and extrinsics created by `typegen`.
-* `src/mappings` - mapping module.
-* `lib` - compiled js files. The structure of this directory must reflect `src`.
-* `.env` - hydra tools are heavily driven by environment variables defined here or supplied by a shell.
+# 6. Open a separate terminal and launch the graphql server to query the processed data
+just serve
+
+# 7. Visit localhost:4350/graphql to see the result
+```
 
 ## Dev flow
 
 ### 1. Define database schema
 
 Start development by defining the schema of the target database via `schema.graphql`.
 Schema definition consists of regular graphql type declarations annotated with custom directives.
-Full description of `schema.graphql` dialect is available [here](https://docs.subsquid.io/schema-file).
+A full description of `schema.graphql` dialect is available [here](https://docs.subsquid.io/schema-file).
 
 ### 2. Generate TypeORM classes
 
-Mapping developers use [TypeORM](https://typeorm.io) entities
-to interact with the target database during data processing. All necessary entity classes are
-generated by the squid framework from `schema.graphql`. This is done by running `npx squid-typeorm-codegen`
-or (equivalently) `sqd codegen` command.
+Mapping developers use [TypeORM](https://typeorm.io) entities to interact with the target database during data processing. The squid framework generates All necessary entity classes from `schema.graphql`. This is done by running `just codegen` command.
 
 ### 3. Generate database migration
 
 All database changes are applied through migration files located at `db/migrations`.
-`squid-typeorm-migration(1)` tool provides several commands to drive the process.
+`squid-typeorm-migration` tool provides several commands to drive the process.
 It is all [TypeORM](https://typeorm.io/#/migrations) under the hood.
 
 ```bash
-# Connect to database, analyze its state and generate migration to match the target schema.
-# The target schema is derived from entity classes generated earlier.
-# Don't forget to compile your entity classes beforehand!
-npx squid-typeorm-migration generate
+# Connect to the database, analyze its state, and generate a migration to match the target schema.
+# Launch Docker instance of the database
+just upd
 
-# Create template file for custom database changes
-npx squid-typeorm-migration create
+# The target schema is derived from entity classes generated earlier.
+# Remember to compile your entity classes beforehand!
+just update-db
 
 # Apply database migrations from `db/migrations`
-npx squid-typeorm-migration apply
+just migrate
 
 # Revert the last performed migration
-npx squid-typeorm-migration revert         
+just revert-db
 ```
+
 Available `sqd` shortcuts:
+
 ```bash
 # Build the project, remove any old migrations, then run `npx squid-typeorm-migration generate`
 sqd migration:generate
@@ -89,7 +94,38 @@ sqd migration:generate
 sqd migration:apply
 ```
 
-### Dev hacks 
+### Testing
+
+> Unit test early, unit test often
+
+> [!NOTE]
+> Any code imported from @kodadot [packages has unit test written in the separated repository](https://github.com/kodadot/packages)
+
+This indexer contains unit tests for utility/parsing functions we wrote.
+
+Tests are located in the `tests/` directory.
+To run the tests, use:
+
+```bash
+npm run test
+```
+
+> [!WARNING]
+> Currently, it is impossible to unit test the whole indexer workflow as a dry run. If you encounter some problem, please head over to the telegram group **HydraDevs**
+
+## Architecture
+
+The architecture of this project is following:
+
+* `src/processable.ts` - definition of Events and Extrinsic to be processed by Squid
+* `src/processor.ts` - processor definition
+* `src/mappings/index` - the main function that is called by the processor to process events and extrinsic
+* `src/mappings/<pallet>` - mapping functions for each event and extrinsic
+* `src/mappings/<pallet>/types.ts` - types for each event and extrinsic
+* `src/mappings/<pallet>/getters/<chain>.ts` - transformation functions for each event and extrinsic
+* `src/mappings/utils` - utility functions used by mappings
+
+## Misc
 
 1. fast generate event handlers 
 

db/migrations/1706015961189-Data.js

@@ -0,0 +1,13 @@
+module.exports = class Data1706015961189 {
+    name = 'Data1706015961189'
+
+    async up(db) {
+        await db.query(`ALTER TABLE "collection_entity" ADD "type" character varying(8)`)
+        await db.query(`ALTER TABLE "collection_entity" ADD "settings" jsonb`)
+    }
+
+    async down(db) {
+        await db.query(`ALTER TABLE "collection_entity" DROP COLUMN "type"`)
+        await db.query(`ALTER TABLE "collection_entity" DROP COLUMN "settings"`)
+    }
+}

db/migrations/1706035705537-Data.js

@@ -0,0 +1,11 @@
+module.exports = class Data1706035705537 {
+    name = 'Data1706035705537'
+
+    async up(db) {
+        await db.query(`CREATE TABLE "cache_status" ("id" character varying NOT NULL, "last_block_timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, CONSTRAINT "PK_1001e39eb0aa38d043d96f7f4fa" PRIMARY KEY ("id"))`)
+    }
+
+    async down(db) {
+        await db.query(`DROP TABLE "cache_status"`)
+    }
+}

docker-compose.yml

@@ -11,6 +11,7 @@ services:
     ports:
       - "${DB_PORT}:5432"
       # command: ["postgres", "-c", "log_statement=all"]
-#    volumes:
+    # volumes:
+      # - ./assets/${CHAIN}/init.sql:/docker-entrypoint-initdb.d/init.sql
 #      - ./data/db:/var/lib/postgresql/data
 

justfile

@@ -2,7 +2,7 @@ default := 'squid'
 types := 'typegen'
 
 process: build
-	node -r dotenv/config lib/processor.js
+	node --require=dotenv/config lib/processor.js
 
 serve:
 	@npx squid-graphql-server
@@ -42,9 +42,16 @@ bug: down upd
 
 reset: migrate
 
+slow: 
+	sleep 1
+	just reset
+
 quickstart: migrate process
 
-quick: build reset process
+quick: wipe bug slow process
+
+wipe:
+  clear
 
 prod TAG:
 	gh pr create --base release-{{TAG}}
@@ -55,6 +62,9 @@ migrate:
 update-db:
 	npx squid-typeorm-migration generate
 
+revert-db:
+	npx squid-typeorm-migration revert         
+
 db: update-db migrate
 
 test:
@@ -79,7 +89,10 @@ update-deps:
 	npx npm-check-updates -ux
 
 exec:
-	docker exec -it stick-db-1 psql -U postgres -d squid
+	docker exec -it subsquid_db psql -U postgres -d squid
+
+dump NAME=default:
+	docker exec -t subsquid_db pg_dump -U postgres -d squid > {{NAME}}.sql
 
 check: codegen build