From 05a7c4619237cbd6b3577087d43646213fd88565 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Wed, 5 Jul 2023 17:22:53 -0500 Subject: [PATCH] Add migrations for handmade indices (#1266) * add indices * add record index to bsky * sync-up bsky index migration, remove duplicate index * backpressure on bsky backfill indexing (#1268) * backpressure on bsky backfill indexing * skip image resolution for text labeler * increase background queue concurrency for backfill * tidy * Proxy timeline skeleton construction (#1264) proxy timeline skeleton construction to appview * Only pass through known params on timeline skeleton (#1270) only pass through own params * Require headers on getRecord proxy (#1271) require headers on getRecord proxy * Add boolean for enabling generic appview proxying (#1273) * add boolean config for enabling generic proxying * tweak * tweak cfg var name * tweak * :bug: Only ignore reports for specific at-uri when ignoreSubject contains at-uri (#1251) * Move timeline construction to appview (#1274) full switch timeline construction to appview * Better propagate errors on repo streams (#1276) better propgate errors on repo streams * Log pds sequencer leader stats (#1277) log pds sequencer leader stats * Explicit dns servers (#1281) * add ability to setup explicit dns servers * cleanup * fix * reorder * pr feedback * Thread through id-resolver cfg (#1282) thread through id-resolver-cfg * Bsky log commit error (#1275) * don't bail on bad record index * add build * temporarily disable check, full reindex on rabase * don't bail on bad record index during rebase, track last commit on rebase * log bsky repo subscription stats * add running and waiting count to repo sub stats * re-enable fast path for happy rebases * only hold onto seq in cursor consecutivelist, don't hold onto whole completed messages * Misc scaling (#1284) * limit backsearch to 1 day instead of 3 * lower like count threshold * bump to 6 * disable like count check * disable with friends * preemptively cache last commit * inline list mutes * actor service * label cache * placehodler on popular with friends * bulk sequence * no limit but chunk * bump chunk to 5k * try 10k * fix notify * tweaking * syntax * one more fix * increase backfill allowance * full refresh label cache * limit 1 on mute list * reserve aclu handle * clean up testing with label cache * note on with-friends * rm defer from label cache * label cache error handling * rm branch build * build appview * update indices --------- Co-authored-by: Devin Ivy Co-authored-by: Foysal Ahamed --- .../workflows/build-and-push-bsky-aws.yaml | 1 - ...30703T045536691Z-feed-and-label-indices.ts | 31 +++++++++++++++++++ packages/bsky/src/db/migrations/index.ts | 1 + ...30703T044601833Z-feed-and-label-indices.ts | 19 ++++++++++++ packages/pds/src/db/migrations/index.ts | 1 + 5 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 packages/bsky/src/db/migrations/20230703T045536691Z-feed-and-label-indices.ts create mode 100644 packages/pds/src/db/migrations/20230703T044601833Z-feed-and-label-indices.ts diff --git a/.github/workflows/build-and-push-bsky-aws.yaml b/.github/workflows/build-and-push-bsky-aws.yaml index 9e5c2e9b870..b656dec89c9 100644 --- a/.github/workflows/build-and-push-bsky-aws.yaml +++ b/.github/workflows/build-and-push-bsky-aws.yaml @@ -3,7 +3,6 @@ on: push: branches: - main - - bsky-log-commit-error env: REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }} USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }} diff --git a/packages/bsky/src/db/migrations/20230703T045536691Z-feed-and-label-indices.ts b/packages/bsky/src/db/migrations/20230703T045536691Z-feed-and-label-indices.ts new file mode 100644 index 00000000000..bf54deeae2a --- /dev/null +++ b/packages/bsky/src/db/migrations/20230703T045536691Z-feed-and-label-indices.ts @@ -0,0 +1,31 @@ +import { Kysely } from 'kysely' + +export async function up(db: Kysely): Promise { + await db.schema + .createIndex('label_cts_idx') + .on('label') + .column('cts') + .execute() + await db.schema.dropIndex('feed_item_originator_idx').execute() + await db.schema + .createIndex('feed_item_originator_cursor_idx') + .on('feed_item') + .columns(['originatorDid', 'sortAt', 'cid']) + .execute() + await db.schema + .createIndex('record_did_idx') + .on('record') + .column('did') + .execute() +} + +export async function down(db: Kysely): Promise { + await db.schema.dropIndex('label_cts_idx').execute() + await db.schema.dropIndex('feed_item_originator_cursor_idx').execute() + await db.schema + .createIndex('feed_item_originator_idx') + .on('feed_item') + .column('originatorDid') + .execute() + await db.schema.dropIndex('record_did_idx').execute() +} diff --git a/packages/bsky/src/db/migrations/index.ts b/packages/bsky/src/db/migrations/index.ts index 487e61b7cf5..6d29ffe6576 100644 --- a/packages/bsky/src/db/migrations/index.ts +++ b/packages/bsky/src/db/migrations/index.ts @@ -19,3 +19,4 @@ export * as _20230611T215300060Z from './20230611T215300060Z-actor-state' export * as _20230620T161134972Z from './20230620T161134972Z-post-langs' export * as _20230627T212437895Z from './20230627T212437895Z-optional-handle' export * as _20230629T220835893Z from './20230629T220835893Z-remove-post-hierarchy' +export * as _20230703T045536691Z from './20230703T045536691Z-feed-and-label-indices' diff --git a/packages/pds/src/db/migrations/20230703T044601833Z-feed-and-label-indices.ts b/packages/pds/src/db/migrations/20230703T044601833Z-feed-and-label-indices.ts new file mode 100644 index 00000000000..530b833c757 --- /dev/null +++ b/packages/pds/src/db/migrations/20230703T044601833Z-feed-and-label-indices.ts @@ -0,0 +1,19 @@ +import { Kysely } from 'kysely' + +export async function up(db: Kysely): Promise { + await db.schema + .createIndex('label_cts_idx') + .on('label') + .column('cts') + .execute() + await db.schema + .createIndex('feed_item_originator_cursor_idx') + .on('feed_item') + .columns(['originatorDid', 'sortAt', 'cid']) + .execute() +} + +export async function down(db: Kysely): Promise { + await db.schema.dropIndex('label_cts_idx').execute() + await db.schema.dropIndex('feed_item_originator_cursor_idx').execute() +} diff --git a/packages/pds/src/db/migrations/index.ts b/packages/pds/src/db/migrations/index.ts index f723e73119f..05bbb8c6fcc 100644 --- a/packages/pds/src/db/migrations/index.ts +++ b/packages/pds/src/db/migrations/index.ts @@ -53,3 +53,4 @@ export * as _20230523T183902064Z from './20230523T183902064Z-algo-whats-hot-view export * as _20230529T222706121Z from './20230529T222706121Z-suggested-follows' export * as _20230530T213530067Z from './20230530T213530067Z-rebase-indices' export * as _20230605T235529700Z from './20230605T235529700Z-outgoing-repo-seq' +export * as _20230703T044601833Z from './20230703T044601833Z-feed-and-label-indices'