Skip to content

Commit

Permalink
refactor: remove old token metadata processor (#1747)
Browse files Browse the repository at this point in the history
* refactor: remove old processor, start fetching data from contract

* refactor: remove old tests

* fix: tests and lint

* fix: use lru cache

* chore: delete old tables

* chore: fix comments [skip ci]

* fix: env comment block [skip ci]

* chore: avoid duplicate RPC lookups

* fix: make an incremental migration to drop tables

* fix: re-add metadata processing ENV flag

---------

Co-authored-by: Matthew Little <[email protected]>
  • Loading branch information
rafaelcr and zone117x committed Nov 13, 2023
1 parent 536dbd1 commit 9cbb678
Show file tree
Hide file tree
Showing 42 changed files with 444 additions and 3,121 deletions.
35 changes: 9 additions & 26 deletions .env
Original file line number Diff line number Diff line change
Expand Up @@ -122,34 +122,17 @@ STACKS_NODE_TYPE=L1
# Override the default file path for the proxy cache control file
# STACKS_API_PROXY_CACHE_CONTROL_FILE=/path/to/.proxy-cache-control.json

# Enable token metadata processing. Disabled by default.
# Enable FT metadata processing for Rosetta operations display. Disabled by default.
# STACKS_API_ENABLE_FT_METADATA=1
# STACKS_API_ENABLE_NFT_METADATA=1

# If token metadata processing is enabled, this variable determines how the API reacts to metadata processing failures.
# When strict mode is enabled, any failures caused by recoverable errors will be retried indefinitely. Otherwise,
# the API will give up after `STACKS_API_TOKEN_METADATA_MAX_RETRIES` is reached for that smart contract.
# STACKS_API_TOKEN_METADATA_STRICT_MODE=1

# Maximum number of times we'll try processing FT/NFT metadata for a specific smart contract if we've failed
# because of a recoverable error.
# Only used if `STACKS_API_TOKEN_METADATA_STRICT_MODE` is disabled.
# STACKS_API_TOKEN_METADATA_MAX_RETRIES=5

# Controls the token metadata error handling mode. The possible values are:
# * `warning`: If required metadata is not found, the API will issue a warning and not display data for that token.
# * `error`: If required metadata is not found, the API will throw an error.
# If not specified or any other value is provided, the mode will be set to `warning`.
# STACKS_API_TOKEN_METADATA_ERROR_MODE=warning

# Configure a script to handle image URLs during token metadata processing.
# This example script uses the `imgix.net` service to create CDN URLs.
# Must be an executable script that accepts the URL as the first program argument
# and outputs a result URL to stdout.
# STACKS_API_IMAGE_CACHE_PROCESSOR=./config/token-metadata-image-cache-imgix.js
# Env vars needed for the above sample `imgix` script:
# IMGIX_DOMAIN=https://<your domain>.imgix.net
# IMGIX_TOKEN=<your token>
# The Rosetta API endpoints require FT metadata to display operations with the proper `symbol` and
# `decimals` values. If FT metadata is enabled, this variable controls the token metadata error
# handling mode when metadata is not found.
# The possible values are:
# * `warning`: The API will issue a warning and not display data for that token.
# * `error`: The API will throw an error. If not specified or any other value is provided, the mode
# will be set to `warning`.
# STACKS_API_TOKEN_METADATA_ERROR_MODE=warning

# Web Socket ping interval to determine client availability, in seconds.
# STACKS_API_WS_PING_INTERVAL=5
Expand Down
91 changes: 0 additions & 91 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -848,96 +848,6 @@ jobs:
flag-name: run-${{ github.job }}
parallel: true

test-tokens-metadata:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2

- name: Use Node.js
uses: actions/setup-node@v2
with:
node-version-file: ".nvmrc"

- name: Install deps
run: npm ci

- name: Setup env vars
run: echo "STACKS_CORE_EVENT_HOST=http://0.0.0.0" >> $GITHUB_ENV

- name: Setup integration environment
run: |
sudo ufw disable
npm run devenv:deploy-krypton -- -d
npm run devenv:logs-krypton -- --no-color &> docker-compose-logs.txt &
- name: Run tokens tests
run: npm run test:tokens-metadata

- name: Print integration environment logs
run: cat docker-compose-logs.txt
if: failure()

- name: Teardown integration environment
run: npm run devenv:stop-krypton
if: always()

- name: Upload coverage to Codecov
uses: codecov/codecov-action@v2
if: always()

- name: Upload coverage to Coveralls
uses: coverallsapp/github-action@master
if: ${{ false }}
with:
github-token: ${{ secrets.github_token }}
flag-name: run-${{ github.job }}
parallel: true

test-tokens-strict:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2

- name: Use Node.js
uses: actions/setup-node@v2
with:
node-version-file: ".nvmrc"

- name: Install deps
run: npm ci

- name: Setup env vars
run: echo "STACKS_CORE_EVENT_HOST=http://0.0.0.0" >> $GITHUB_ENV

- name: Setup integration environment
run: |
sudo ufw disable
npm run devenv:deploy-krypton -- -d
npm run devenv:logs-krypton -- --no-color &> docker-compose-logs.txt &
- name: Run tokens tests
run: npm run test:tokens-strict

- name: Print integration environment logs
run: cat docker-compose-logs.txt
if: failure()

- name: Teardown integration environment
run: npm run devenv:stop-krypton
if: always()

- name: Upload coverage to Codecov
uses: codecov/codecov-action@v2
if: always()

- name: Upload coverage to Coveralls
uses: coverallsapp/github-action@master
if: ${{ false }}
with:
github-token: ${{ secrets.github_token }}
flag-name: run-${{ github.job }}
parallel: true

build-publish:
runs-on: ubuntu-latest
needs:
Expand All @@ -948,7 +858,6 @@ jobs:
- test-bns
- test-rosetta
- test-rosetta-cli-construction
- test-tokens-strict
steps:
- uses: actions/checkout@v2
with:
Expand Down
34 changes: 0 additions & 34 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -346,40 +346,6 @@
"preLaunchTask": "deploy:krypton",
"postDebugTask": "stop:krypton",
},
{
"type": "node",
"request": "launch",
"name": "Jest: Tokens - strict mode",
"program": "${workspaceFolder}/node_modules/.bin/jest",
"args": [
"--testTimeout=3600000",
"--runInBand",
"--no-cache",
"--config",
"${workspaceRoot}/tests/jest.config.tokens-strict.js",
],
"outputCapture": "std",
"console": "integratedTerminal",
"preLaunchTask": "deploy:krypton",
"postDebugTask": "stop:krypton",
},
{
"type": "node",
"request": "launch",
"name": "Jest: Tokens - metadata",
"program": "${workspaceFolder}/node_modules/.bin/jest",
"args": [
"--testTimeout=3600000",
"--runInBand",
"--no-cache",
"--config",
"${workspaceRoot}/tests/jest.config.tokens-metadata.js",
],
"outputCapture": "std",
"console": "integratedTerminal",
"preLaunchTask": "deploy:krypton",
"postDebugTask": "stop:krypton",
},
{
"type": "node",
"request": "launch",
Expand Down
132 changes: 132 additions & 0 deletions migrations/1699540187362_remove-token-metadata.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
/* eslint-disable camelcase */

exports.shorthands = undefined;

exports.up = pgm => {
pgm.dropTable('token_metadata_queue');
pgm.dropTable('nft_metadata');
pgm.dropTable('ft_metadata');
};

exports.down = pgm => {
pgm.createTable('token_metadata_queue', {
queue_id: {
type: 'serial',
primaryKey: true,
},
tx_id: {
type: 'bytea',
notNull: true,
},
contract_id: {
type: 'string',
notNull: true,
},
contract_abi: {
type: 'string',
notNull: true,
},
block_height: {
type: 'integer',
notNull: true,
},
processed: {
type: 'boolean',
notNull: true,
},
retry_count: {
type: 'integer',
notNull: true,
default: 0,
}
});
pgm.createIndex('token_metadata_queue', [{ name: 'block_height', sort: 'DESC' }]);
pgm.createTable('nft_metadata', {
id: {
type: 'serial',
primaryKey: true,
},
name: {
type: 'string',
notNull: true,
},
token_uri: {
type: 'string',
notNull: true,
},
description: {
type: 'string',
notNull: true,
},
image_uri: {
type: 'string',
notNull: true,
},
image_canonical_uri: {
type: 'string',
notNull: true,
},
contract_id: {
type: 'string',
notNull: true,
unique: true,
},
tx_id: {
type: 'bytea',
notNull: true,
},
sender_address: {
type: 'string',
notNull: true,
}
});
pgm.createIndex('nft_metadata', 'contract_id', { method: 'hash' });
pgm.createTable('ft_metadata', {
id: {
type: 'serial',
primaryKey: true,
},
name: {
type: 'string',
notNull: true,
},
token_uri: {
type: 'string',
notNull: true,
},
description: {
type: 'string',
notNull: true,
},
image_uri: {
type: 'string',
notNull: true,
},
image_canonical_uri: {
type: 'string',
notNull: true,
},
contract_id: {
type: 'string',
notNull: true,
unique: true,
},
symbol: {
type: 'string',
notNull: true,
},
decimals: {
type: 'integer',
notNull: true,
},
tx_id: {
type: 'bytea',
notNull: true,
},
sender_address: {
type: 'string',
notNull: true,
}
});
pgm.createIndex('ft_metadata', 'contract_id', { method: 'hash' });
}
2 changes: 0 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@
"test:integration:rosetta-cli:construction": "concurrently \"npm:devenv:deploy-krypton\" \"cross-env NODE_ENV=development STACKS_CHAIN_ID=0x80000000 jest --config ./tests/jest.config.rosetta-cli-construction.js --no-cache --runInBand; npm run devenv:stop-krypton\"",
"test:integration:bns": "concurrently \"npm:devenv:deploy-krypton\" \"cross-env NODE_ENV=development jest --config ./tests/jest.config.bns.js --no-cache --runInBand; npm run devenv:stop-krypton\"",
"test:integration:bns-e2e": "concurrently \"npm:devenv:deploy-krypton\" \"cross-env NODE_ENV=development jest --config ./tests/jest.config.bns-e2e.js --no-cache --runInBand; npm run devenv:stop-krypton\"",
"test:integration:tokens-strict": "concurrently \"npm:devenv:deploy-krypton\" \"cross-env NODE_ENV=development jest --config ./tests/jest.config.tokens-strict.js --no-cache --runInBand; npm run devenv:stop-krypton\"",
"test:integration:tokens-metadata": "concurrently \"npm:devenv:deploy-krypton\" \"cross-env NODE_ENV=development jest --config ./tests/jest.config.tokens-metadata.js --no-cache --runInBand; npm run devenv:stop-krypton\"",
"test:integration:rpc": "concurrently \"npm:devenv:deploy-krypton\" \"cross-env NODE_ENV=development jest --config ./tests/jest.config.rpc.js --no-cache --runInBand; npm run devenv:stop-krypton\"",
"test:integration:event-replay": "concurrently \"docker compose -f docker/docker-compose.dev.postgres.yml up --force-recreate -V\" \"cross-env NODE_ENV=development jest --config ./tests/jest.config.event-replay.js --no-cache --runInBand; npm run devenv:stop\"",
"test:integration:btc-faucet": "concurrently \"docker compose -f docker/docker-compose.dev.postgres.yml -f docker/docker-compose.dev.bitcoind.yml up --force-recreate -V\" \"cross-env NODE_ENV=development jest --config ./tests/jest.config.btc-faucet.js --no-cache --runInBand; npm run devenv:stop\"",
Expand Down
26 changes: 1 addition & 25 deletions src/api/controllers/db-controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ import {
} from '../../datastore/common';
import { unwrapOptional, FoundOrNot, unixEpochToIso, EMPTY_HASH_256, ChainID } from '../../helpers';
import { serializePostCondition, serializePostConditionMode } from '../serializers/post-conditions';
import { getOperations, parseTransactionMemo } from '../../rosetta-helpers';
import { getOperations, parseTransactionMemo } from '../../rosetta/rosetta-helpers';
import { PgStore } from '../../datastore/pg-store';
import { Pox2EventName } from '../../pox-helpers';
import { logger } from '../../logger';
Expand Down Expand Up @@ -164,30 +164,6 @@ export function getTxStatus(txStatus: DbTxStatus | string): string {
}
}

type EventTypeString =
| 'smart_contract_log'
| 'stx_asset'
| 'fungible_token_asset'
| 'non_fungible_token_asset'
| 'stx_lock';

export function getEventTypeString(eventTypeId: DbEventTypeId): EventTypeString {
switch (eventTypeId) {
case DbEventTypeId.SmartContractLog:
return 'smart_contract_log';
case DbEventTypeId.StxAsset:
return 'stx_asset';
case DbEventTypeId.FungibleTokenAsset:
return 'fungible_token_asset';
case DbEventTypeId.NonFungibleTokenAsset:
return 'non_fungible_token_asset';
case DbEventTypeId.StxLock:
return 'stx_lock';
default:
throw new Error(`Unexpected DbEventTypeId: ${eventTypeId}`);
}
}

export function getAssetEventTypeString(
assetEventTypeId: DbAssetEventTypeId
): 'transfer' | 'mint' | 'burn' {
Expand Down
2 changes: 1 addition & 1 deletion src/api/init.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import * as expressListEndpoints from 'express-list-endpoints';
import { createMiddleware as createPrometheusMiddleware } from '@promster/express';
import { createMicroblockRouter } from './routes/microblock';
import { createStatusRouter } from './routes/status';
import { createTokenRouter } from './routes/tokens/tokens';
import { createTokenRouter } from './routes/tokens';
import { createFeeRateRouter } from './routes/fee-rate';
import { setResponseNonCacheable } from './controllers/cache-controller';

Expand Down
Loading

0 comments on commit 9cbb678

Please sign in to comment.