From 26c69697703be4a0138c3cb35588971b47549b68 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Thu, 26 Sep 2024 16:46:47 -0600 Subject: [PATCH 1/4] feat: add principal cache --- src/api/controllers/cache-controller.ts | 83 +++++-------- src/api/routes/address.ts | 21 ++-- src/api/routes/v2/addresses.ts | 9 +- src/datastore/pg-store.ts | 40 +++++++ tests/api/cache-control.test.ts | 147 +++++++++++++++++------- 5 files changed, 190 insertions(+), 110 deletions(-) diff --git a/src/api/controllers/cache-controller.ts b/src/api/controllers/cache-controller.ts index e6ee858a5..dd0ee5176 100644 --- a/src/api/controllers/cache-controller.ts +++ b/src/api/controllers/cache-controller.ts @@ -2,17 +2,13 @@ import * as prom from 'prom-client'; import { normalizeHashString } from '../../helpers'; import { PgStore } from '../../datastore/pg-store'; import { logger } from '../../logger'; -import { sha256 } from '@hirosystems/api-toolkit'; +import { + CACHE_CONTROL_MUST_REVALIDATE, + parseIfNoneMatchHeader, + sha256, +} from '@hirosystems/api-toolkit'; import { FastifyReply, FastifyRequest } from 'fastify'; -/** - * A `Cache-Control` header used for re-validation based caching. - * * `public` == allow proxies/CDNs to cache as opposed to only local browsers. - * * `no-cache` == clients can cache a resource but should revalidate each time before using it. - * * `must-revalidate` == somewhat redundant directive to assert that cache must be revalidated, required by some CDNs - */ -const CACHE_CONTROL_MUST_REVALIDATE = 'public, no-cache, must-revalidate'; - /** * Describes a key-value to be saved into a request's locals, representing the current * state of the chain depending on the type of information being requested by the endpoint. @@ -25,6 +21,8 @@ enum ETagType { mempool = 'mempool', /** ETag based on the status of a single transaction across the mempool or canonical chain. */ transaction = 'transaction', + /** Etag based on the confirmed balance of a single principal (STX address or contract id) */ + principal = 'principal', } /** Value that means the ETag did get calculated but it is empty. */ @@ -75,52 +73,6 @@ function getETagMetrics(): ETagCacheMetrics { return _eTagMetrics; } -/** - * Parses the etag values from a raw `If-None-Match` request header value. - * The wrapping double quotes (if any) and validation prefix (if any) are stripped. - * The parsing is permissive to account for commonly non-spec-compliant clients, proxies, CDNs, etc. - * E.g. the value: - * ```js - * `"a", W/"b", c,d, "e", "f"` - * ``` - * Would be parsed and returned as: - * ```js - * ['a', 'b', 'c', 'd', 'e', 'f'] - * ``` - * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match#syntax - * ``` - * If-None-Match: "etag_value" - * If-None-Match: "etag_value", "etag_value", ... - * If-None-Match: * - * ``` - * @param ifNoneMatchHeaderValue - raw header value - * @returns an array of etag values - */ -export function parseIfNoneMatchHeader( - ifNoneMatchHeaderValue: string | undefined -): string[] | undefined { - if (!ifNoneMatchHeaderValue) { - return undefined; - } - // Strip wrapping double quotes like `"hello"` and the ETag validation-prefix like `W/"hello"`. - // The API returns compliant, strong-validation ETags (double quoted ASCII), but can't control what - // clients, proxies, CDNs, etc may provide. - const normalized = /^(?:"|W\/")?(.*?)"?$/gi.exec(ifNoneMatchHeaderValue.trim())?.[1]; - if (!normalized) { - // This should never happen unless handling a buggy request with something like `If-None-Match: ""`, - // or if there's a flaw in the above code. Log warning for now. - logger.warn(`Normalized If-None-Match header is falsy: ${ifNoneMatchHeaderValue}`); - return undefined; - } else if (normalized.includes(',')) { - // Multiple etag values provided, likely irrelevant extra values added by a proxy/CDN. - // Split on comma, also stripping quotes, weak-validation prefixes, and extra whitespace. - return normalized.split(/(?:W\/"|")?(?:\s*),(?:\s*)(?:W\/"|")?/gi); - } else { - // Single value provided (the typical case) - return [normalized]; - } -} - async function calculateETag( db: PgStore, etagType: ETagType, @@ -155,7 +107,7 @@ async function calculateETag( } return digest.result.digest; } catch (error) { - logger.error(error, 'Unable to calculate mempool'); + logger.error(error, 'Unable to calculate mempool etag'); return; } @@ -178,7 +130,20 @@ async function calculateETag( ]; return sha256(elements.join(':')); } catch (error) { - logger.error(error, 'Unable to calculate transaction'); + logger.error(error, 'Unable to calculate transaction etag'); + return; + } + + case ETagType.principal: + try { + const params = req.params as { address?: string; principal?: string }; + const principal = params.address ?? params.principal; + if (!principal) return ETAG_EMPTY; + const activity = await db.getPrincipalLastActivityTxIds(principal); + const text = `${activity.stx_tx_id}:${activity.ft_tx_id}:${activity.nft_tx_id}`; + return sha256(text); + } catch (error) { + logger.error(error, 'Unable to calculate principal etag'); return; } } @@ -224,3 +189,7 @@ export async function handleMempoolCache(request: FastifyRequest, reply: Fastify export async function handleTransactionCache(request: FastifyRequest, reply: FastifyReply) { return handleCache(ETagType.transaction, request, reply); } + +export async function handlePrincipalCache(request: FastifyRequest, reply: FastifyReply) { + return handleCache(ETagType.principal, request, reply); +} diff --git a/src/api/routes/address.ts b/src/api/routes/address.ts index ad66b7cdc..6b38f3079 100644 --- a/src/api/routes/address.ts +++ b/src/api/routes/address.ts @@ -15,7 +15,12 @@ import { } from '../controllers/db-controller'; import { InvalidRequestError, InvalidRequestErrorType, NotFoundError } from '../../errors'; import { decodeClarityValueToRepr } from 'stacks-encoding-native-js'; -import { handleChainTipCache, handleMempoolCache } from '../controllers/cache-controller'; +import { + handleChainTipCache, + handleMempoolCache, + handlePrincipalCache, + handleTransactionCache, +} from '../controllers/cache-controller'; import { PgStore } from '../../datastore/pg-store'; import { logger } from '../../logger'; import { has0xPrefix } from '@hirosystems/api-toolkit'; @@ -86,7 +91,7 @@ export const AddressRoutes: FastifyPluginAsync< fastify.get( '/:principal/stx', { - preHandler: handleChainTipCache, + preHandler: handlePrincipalCache, schema: { operationId: 'get_account_stx_balance', summary: 'Get account STX balance', @@ -142,7 +147,7 @@ export const AddressRoutes: FastifyPluginAsync< fastify.get( '/:principal/balances', { - preHandler: handleChainTipCache, + preHandler: handlePrincipalCache, schema: { operationId: 'get_account_balance', summary: 'Get account balances', @@ -234,7 +239,7 @@ export const AddressRoutes: FastifyPluginAsync< fastify.get( '/:principal/transactions', { - preHandler: handleChainTipCache, + preHandler: handlePrincipalCache, schema: { deprecated: true, operationId: 'get_account_transactions', @@ -307,7 +312,7 @@ export const AddressRoutes: FastifyPluginAsync< fastify.get( '/:principal/:tx_id/with_transfers', { - preHandler: handleChainTipCache, + preHandler: handleTransactionCache, schema: { deprecated: true, operationId: 'get_single_transaction_with_transfers', @@ -373,7 +378,7 @@ export const AddressRoutes: FastifyPluginAsync< fastify.get( '/:principal/transactions_with_transfers', { - preHandler: handleChainTipCache, + preHandler: handlePrincipalCache, schema: { deprecated: true, operationId: 'get_account_transactions_with_transfers', @@ -485,7 +490,7 @@ export const AddressRoutes: FastifyPluginAsync< fastify.get( '/:principal/assets', { - preHandler: handleChainTipCache, + preHandler: handlePrincipalCache, schema: { operationId: 'get_account_assets', summary: 'Get account assets', @@ -533,7 +538,7 @@ export const AddressRoutes: FastifyPluginAsync< fastify.get( '/:principal/stx_inbound', { - preHandler: handleChainTipCache, + preHandler: handlePrincipalCache, schema: { operationId: 'get_account_inbound', summary: 'Get inbound STX transfers', diff --git a/src/api/routes/v2/addresses.ts b/src/api/routes/v2/addresses.ts index 62410b9cc..f36d66582 100644 --- a/src/api/routes/v2/addresses.ts +++ b/src/api/routes/v2/addresses.ts @@ -1,4 +1,7 @@ -import { handleChainTipCache } from '../../../api/controllers/cache-controller'; +import { + handlePrincipalCache, + handleTransactionCache, +} from '../../../api/controllers/cache-controller'; import { AddressParamsSchema, AddressTransactionParamsSchema } from './schemas'; import { parseDbAddressTransactionTransfer, parseDbTxWithAccountTransferSummary } from './helpers'; import { InvalidRequestError, NotFoundError } from '../../../errors'; @@ -23,7 +26,7 @@ export const AddressRoutesV2: FastifyPluginAsync< fastify.get( '/:address/transactions', { - preHandler: handleChainTipCache, + preHandler: handlePrincipalCache, schema: { operationId: 'get_address_transactions', summary: 'Get address transactions', @@ -71,7 +74,7 @@ export const AddressRoutesV2: FastifyPluginAsync< fastify.get( '/:address/transactions/:tx_id/events', { - preHandler: handleChainTipCache, + preHandler: handleTransactionCache, schema: { operationId: 'get_address_transaction_events', summary: 'Get events for an address transaction', diff --git a/src/datastore/pg-store.ts b/src/datastore/pg-store.ts index 9cc7b62c0..9a494adb3 100644 --- a/src/datastore/pg-store.ts +++ b/src/datastore/pg-store.ts @@ -4404,4 +4404,44 @@ export class PgStore extends BasePgStore { } return result; } + + /** Retrieves the last transaction IDs with STX, FT and NFT activity for a principal */ + async getPrincipalLastActivityTxIds( + principal: string + ): Promise<{ stx_tx_id: string | null; ft_tx_id: string | null; nft_tx_id: string | null }> { + const result = await this.sql< + { stx_tx_id: string | null; ft_tx_id: string | null; nft_tx_id: string | null }[] + >` + WITH last_stx AS ( + SELECT tx_id + FROM principal_stx_txs + WHERE principal = ${principal} AND canonical = true AND microblock_canonical = true + ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC + LIMIT 1 + ), + last_ft AS ( + SELECT tx_id + FROM ft_events + WHERE (sender = ${principal} OR recipient = ${principal}) + AND canonical = true + AND microblock_canonical = true + ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC, event_index DESC + LIMIT 1 + ), + last_nft AS ( + SELECT tx_id + FROM nft_events + WHERE (sender = ${principal} OR recipient = ${principal}) + AND canonical = true + AND microblock_canonical = true + ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC, event_index DESC + LIMIT 1 + ) + SELECT + (SELECT tx_id FROM last_stx) AS stx_tx_id, + (SELECT tx_id FROM last_ft) AS ft_tx_id, + (SELECT tx_id FROM last_nft) AS nft_tx_id + `; + return result[0]; + } } diff --git a/tests/api/cache-control.test.ts b/tests/api/cache-control.test.ts index 92b30122a..dcb88c989 100644 --- a/tests/api/cache-control.test.ts +++ b/tests/api/cache-control.test.ts @@ -10,7 +10,6 @@ import { } from '../../src/datastore/common'; import { startApiServer, ApiServer } from '../../src/api/init'; import { I32_MAX } from '../../src/helpers'; -import { parseIfNoneMatchHeader } from '../../src/api/controllers/cache-controller'; import { TestBlockBuilder, testMempoolTx } from '../utils/test-builders'; import { PgWriteStore } from '../../src/datastore/pg-write-store'; import { PgSqlClient, bufferToHex } from '@hirosystems/api-toolkit'; @@ -38,47 +37,6 @@ describe('cache-control tests', () => { await migrate('down'); }); - test('parse if-none-match header', () => { - // Test various combinations of etags with and without weak-validation prefix, with and without - // wrapping quotes, without and without spaces after commas. - const vectors: { - input: string | undefined; - output: string[] | undefined; - }[] = [ - { input: '""', output: undefined }, - { input: '', output: undefined }, - { input: undefined, output: undefined }, - { - input: '"bfc13a64729c4290ef5b2c2730249c88ca92d82d"', - output: ['bfc13a64729c4290ef5b2c2730249c88ca92d82d'], - }, - { input: 'W/"67ab43", "54ed21", "7892dd"', output: ['67ab43', '54ed21', '7892dd'] }, - { input: '"fail space" ', output: ['fail space'] }, - { input: 'W/"5e15153d-120f"', output: ['5e15153d-120f'] }, - { - input: '"", "" , "asdf"', - output: ['', '', 'asdf'], - }, - { - input: '"","","asdf"', - output: ['', '', 'asdf'], - }, - { - input: 'W/"","","asdf"', - output: ['', '', 'asdf'], - }, - { - input: '"",W/"", W/"asdf", "abcd","123"', - output: ['', '', 'asdf', 'abcd', '123'], - }, - ]; - expect(vectors).toBeTruthy(); - for (const entry of vectors) { - const result = parseIfNoneMatchHeader(entry.input); - expect(result).toEqual(entry.output); - } - }); - test('block chaintip cache control', async () => { const addr1 = 'ST28D4Q6RCQSJ6F7TEYWQDS4N1RXYEP9YBWMYSB97'; const addr2 = 'STB44HYPYAT2BB2QE513NSP81HTMYWBJP02HPGK6'; @@ -644,4 +602,109 @@ describe('cache-control tests', () => { const etag5 = request11.headers['etag']; expect(etag2).not.toBe(etag5); }); + + test('principal cache control', async () => { + const sender_address = 'STB44HYPYAT2BB2QE513NSP81HTMYWBJP02HPGK6'; + const url = `/extended/v2/addresses/${sender_address}/transactions`; + await db.update( + new TestBlockBuilder({ + block_height: 1, + index_block_hash: '0x01', + parent_index_block_hash: '0x00', + }).build() + ); + + // ETag zero. + const request1 = await supertest(api.server).get(url); + expect(request1.status).toBe(200); + expect(request1.type).toBe('application/json'); + const etag0 = request1.headers['etag']; + + // Add STX txs. + await db.update( + new TestBlockBuilder({ + block_height: 2, + index_block_hash: '0x02', + parent_index_block_hash: '0x01', + }) + .addTx({ tx_id: '0x0001', sender_address, token_transfer_amount: 200n }) + .addTxStxEvent({ sender: sender_address, amount: 200n }) + .build() + ); + + // Valid ETag. + const request2 = await supertest(api.server).get(url); + expect(request2.status).toBe(200); + expect(request2.type).toBe('application/json'); + expect(request2.headers['etag']).toBeTruthy(); + const etag1 = request2.headers['etag']; + expect(etag1).not.toEqual(etag0); + + // Cache works with valid ETag. + const request3 = await supertest(api.server).get(url).set('If-None-Match', etag1); + expect(request3.status).toBe(304); + expect(request3.text).toBe(''); + + // Add FT tx. + await db.update( + new TestBlockBuilder({ + block_height: 3, + index_block_hash: '0x03', + parent_index_block_hash: '0x02', + }) + .addTx({ tx_id: '0x0002' }) + .addTxFtEvent({ recipient: sender_address }) + .build() + ); + + // Cache is now a miss. + const request4 = await supertest(api.server).get(url).set('If-None-Match', etag1); + expect(request4.status).toBe(200); + expect(request4.type).toBe('application/json'); + expect(request4.headers['etag'] !== etag1).toEqual(true); + const etag2 = request4.headers['etag']; + + // Cache works with new ETag. + const request5 = await supertest(api.server).get(url).set('If-None-Match', etag2); + expect(request5.status).toBe(304); + expect(request5.text).toBe(''); + + // Add NFT tx. + await db.update( + new TestBlockBuilder({ + block_height: 4, + index_block_hash: '0x04', + parent_index_block_hash: '0x03', + }) + .addTx({ tx_id: '0x0003' }) + .addTxNftEvent({ recipient: sender_address }) + .build() + ); + + // Cache is now a miss. + const request6 = await supertest(api.server).get(url).set('If-None-Match', etag2); + expect(request6.status).toBe(200); + expect(request6.type).toBe('application/json'); + expect(request6.headers['etag'] !== etag1).toEqual(true); + const etag3 = request6.headers['etag']; + + // Cache works with new ETag. + const request7 = await supertest(api.server).get(url).set('If-None-Match', etag3); + expect(request7.status).toBe(304); + expect(request7.text).toBe(''); + + // Advance chain with no changes to this address. + await db.update( + new TestBlockBuilder({ + block_height: 5, + index_block_hash: '0x05', + parent_index_block_hash: '0x04', + }).build() + ); + + // Cache still works. + const request8 = await supertest(api.server).get(url).set('If-None-Match', etag3); + expect(request8.status).toBe(304); + expect(request8.text).toBe(''); + }); }); From cd1a786136c6c0e912c600a65f6678636731f1b4 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Fri, 27 Sep 2024 13:22:52 -0600 Subject: [PATCH 2/4] fix: also consider sponsor transactions --- src/datastore/pg-write-store.ts | 1 + .../importers/new-block-importer.ts | 1 + tests/api/cache-control.test.ts | 35 +++++++++++++++---- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/datastore/pg-write-store.ts b/src/datastore/pg-write-store.ts index 50d864877..a1b79e599 100644 --- a/src/datastore/pg-write-store.ts +++ b/src/datastore/pg-write-store.ts @@ -1180,6 +1180,7 @@ export class PgWriteStore extends PgStore { tx.token_transfer_recipient_address, tx.contract_call_contract_id, tx.smart_contract_contract_id, + tx.sponsor_address, ].filter((p): p is string => !!p) ); for (const event of stxEvents) { diff --git a/src/event-replay/parquet-based/importers/new-block-importer.ts b/src/event-replay/parquet-based/importers/new-block-importer.ts index d34f0c845..7caec4950 100644 --- a/src/event-replay/parquet-based/importers/new-block-importer.ts +++ b/src/event-replay/parquet-based/importers/new-block-importer.ts @@ -251,6 +251,7 @@ const populateBatchInserters = (db: PgWriteStore) => { entry.tx.token_transfer_recipient_address, entry.tx.contract_call_contract_id, entry.tx.smart_contract_contract_id, + entry.tx.sponsor_address, ] .filter((p): p is string => !!p) .forEach(p => principals.add(p)); diff --git a/tests/api/cache-control.test.ts b/tests/api/cache-control.test.ts index dcb88c989..78f06774f 100644 --- a/tests/api/cache-control.test.ts +++ b/tests/api/cache-control.test.ts @@ -661,7 +661,7 @@ describe('cache-control tests', () => { const request4 = await supertest(api.server).get(url).set('If-None-Match', etag1); expect(request4.status).toBe(200); expect(request4.type).toBe('application/json'); - expect(request4.headers['etag'] !== etag1).toEqual(true); + expect(request4.headers['etag']).not.toEqual(etag1); const etag2 = request4.headers['etag']; // Cache works with new ETag. @@ -685,7 +685,7 @@ describe('cache-control tests', () => { const request6 = await supertest(api.server).get(url).set('If-None-Match', etag2); expect(request6.status).toBe(200); expect(request6.type).toBe('application/json'); - expect(request6.headers['etag'] !== etag1).toEqual(true); + expect(request6.headers['etag']).not.toEqual(etag2); const etag3 = request6.headers['etag']; // Cache works with new ETag. @@ -693,18 +693,41 @@ describe('cache-control tests', () => { expect(request7.status).toBe(304); expect(request7.text).toBe(''); - // Advance chain with no changes to this address. + // Add sponsored tx. await db.update( new TestBlockBuilder({ block_height: 5, index_block_hash: '0x05', parent_index_block_hash: '0x04', + }) + .addTx({ tx_id: '0x0004', sponsor_address: sender_address }) + .build() + ); + + // Cache is now a miss. + const request8 = await supertest(api.server).get(url).set('If-None-Match', etag2); + expect(request8.status).toBe(200); + expect(request8.type).toBe('application/json'); + expect(request8.headers['etag']).not.toEqual(etag3); + const etag4 = request8.headers['etag']; + + // Cache works with new ETag. + const request9 = await supertest(api.server).get(url).set('If-None-Match', etag4); + expect(request9.status).toBe(304); + expect(request9.text).toBe(''); + + // Advance chain with no changes to this address. + await db.update( + new TestBlockBuilder({ + block_height: 6, + index_block_hash: '0x06', + parent_index_block_hash: '0x05', }).build() ); // Cache still works. - const request8 = await supertest(api.server).get(url).set('If-None-Match', etag3); - expect(request8.status).toBe(304); - expect(request8.text).toBe(''); + const request10 = await supertest(api.server).get(url).set('If-None-Match', etag4); + expect(request10.status).toBe(304); + expect(request10.text).toBe(''); }); }); From 49bc868cea467a022c0d2bbdb9ab2179519ec75e Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Fri, 27 Sep 2024 13:42:50 -0600 Subject: [PATCH 3/4] chore: import sponsors in migration --- ...727465879167_principal-stx-txs-sponsors.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 migrations/1727465879167_principal-stx-txs-sponsors.js diff --git a/migrations/1727465879167_principal-stx-txs-sponsors.js b/migrations/1727465879167_principal-stx-txs-sponsors.js new file mode 100644 index 000000000..745abbfa2 --- /dev/null +++ b/migrations/1727465879167_principal-stx-txs-sponsors.js @@ -0,0 +1,19 @@ +/* eslint-disable camelcase */ + +exports.shorthands = undefined; + +exports.up = pgm => { + pgm.sql(` + INSERT INTO principal_stx_txs + (principal, tx_id, block_height, index_block_hash, microblock_hash, microblock_sequence, + tx_index, canonical, microblock_canonical) + ( + SELECT + sponsor_address AS principal, tx_id, block_height, index_block_hash, microblock_hash, + microblock_sequence, tx_index, canonical, microblock_canonical + FROM txs + WHERE sponsor_address IS NOT NULL + ) + ON CONFLICT ON CONSTRAINT unique_principal_tx_id_index_block_hash_microblock_hash DO NOTHING + `); +}; From 13fe3979fff21821875c66182b58bf6ff36be750 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Fri, 27 Sep 2024 13:46:52 -0600 Subject: [PATCH 4/4] fix: add down migration --- migrations/1727465879167_principal-stx-txs-sponsors.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/migrations/1727465879167_principal-stx-txs-sponsors.js b/migrations/1727465879167_principal-stx-txs-sponsors.js index 745abbfa2..4b969e2eb 100644 --- a/migrations/1727465879167_principal-stx-txs-sponsors.js +++ b/migrations/1727465879167_principal-stx-txs-sponsors.js @@ -17,3 +17,5 @@ exports.up = pgm => { ON CONFLICT ON CONSTRAINT unique_principal_tx_id_index_block_hash_microblock_hash DO NOTHING `); }; + +exports.down = pgm => {};