Add migrations for handmade indices (#1266)

* add indices * add record index to bsky * sync-up bsky index migration, remove duplicate index * backpressure on bsky backfill indexing (#1268) * backpressure on bsky backfill indexing * skip image resolution for text labeler * increase background queue concurrency for backfill * tidy * Proxy timeline skeleton construction (#1264) proxy timeline skeleton construction to appview * Only pass through known params on timeline skeleton (#1270) only pass through own params * Require headers on getRecord proxy (#1271) require headers on getRecord proxy * Add boolean for enabling generic appview proxying (#1273) * add boolean config for enabling generic proxying * tweak * tweak cfg var name * tweak * 🐛 Only ignore reports for specific at-uri when ignoreSubject contains at-uri (#1251) * Move timeline construction to appview (#1274) full switch timeline construction to appview * Better propagate errors on repo streams (#1276) better propgate errors on repo streams * Log pds sequencer leader stats (#1277) log pds sequencer leader stats * Explicit dns servers (#1281) * add ability to setup explicit dns servers * cleanup * fix * reorder * pr feedback * Thread through id-resolver cfg (#1282) thread through id-resolver-cfg * Bsky log commit error (#1275) * don't bail on bad record index * add build * temporarily disable check, full reindex on rabase * don't bail on bad record index during rebase, track last commit on rebase * log bsky repo subscription stats * add running and waiting count to repo sub stats * re-enable fast path for happy rebases * only hold onto seq in cursor consecutivelist, don't hold onto whole completed messages * Misc scaling (#1284) * limit backsearch to 1 day instead of 3 * lower like count threshold * bump to 6 * disable like count check * disable with friends * preemptively cache last commit * inline list mutes * actor service * label cache * placehodler on popular with friends * bulk sequence * no limit but chunk * bump chunk to 5k * try 10k * fix notify * tweaking * syntax * one more fix * increase backfill allowance * full refresh label cache * limit 1 on mute list * reserve aclu handle * clean up testing with label cache * note on with-friends * rm defer from label cache * label cache error handling * rm branch build * build appview * update indices --------- Co-authored-by: Devin Ivy <[email protected]> Co-authored-by: Foysal Ahamed <[email protected]>
bluesky-social · Jul 5, 2023 · 05a7c46 · 05a7c46
1 parent 4d1f8d3
commit 05a7c46
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 1 deletion.
diff --git a/.github/workflows/build-and-push-bsky-aws.yaml b/.github/workflows/build-and-push-bsky-aws.yaml
@@ -3,7 +3,6 @@ on:
   push:
     branches:
       - main
-      - bsky-log-commit-error
 env:
   REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }}
   USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }}

diff --git a/packages/bsky/src/db/migrations/20230703T045536691Z-feed-and-label-indices.ts b/packages/bsky/src/db/migrations/20230703T045536691Z-feed-and-label-indices.ts
@@ -0,0 +1,31 @@
+import { Kysely } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createIndex('label_cts_idx')
+    .on('label')
+    .column('cts')
+    .execute()
+  await db.schema.dropIndex('feed_item_originator_idx').execute()
+  await db.schema
+    .createIndex('feed_item_originator_cursor_idx')
+    .on('feed_item')
+    .columns(['originatorDid', 'sortAt', 'cid'])
+    .execute()
+  await db.schema
+    .createIndex('record_did_idx')
+    .on('record')
+    .column('did')
+    .execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropIndex('label_cts_idx').execute()
+  await db.schema.dropIndex('feed_item_originator_cursor_idx').execute()
+  await db.schema
+    .createIndex('feed_item_originator_idx')
+    .on('feed_item')
+    .column('originatorDid')
+    .execute()
+  await db.schema.dropIndex('record_did_idx').execute()
+}
diff --git a/packages/bsky/src/db/migrations/index.ts b/packages/bsky/src/db/migrations/index.ts
@@ -19,3 +19,4 @@ export * as _20230611T215300060Z from './20230611T215300060Z-actor-state'
 export * as _20230620T161134972Z from './20230620T161134972Z-post-langs'
 export * as _20230627T212437895Z from './20230627T212437895Z-optional-handle'
 export * as _20230629T220835893Z from './20230629T220835893Z-remove-post-hierarchy'
+export * as _20230703T045536691Z from './20230703T045536691Z-feed-and-label-indices'
diff --git a/packages/pds/src/db/migrations/20230703T044601833Z-feed-and-label-indices.ts b/packages/pds/src/db/migrations/20230703T044601833Z-feed-and-label-indices.ts
@@ -0,0 +1,19 @@
+import { Kysely } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema
+    .createIndex('label_cts_idx')
+    .on('label')
+    .column('cts')
+    .execute()
+  await db.schema
+    .createIndex('feed_item_originator_cursor_idx')
+    .on('feed_item')
+    .columns(['originatorDid', 'sortAt', 'cid'])
+    .execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.dropIndex('label_cts_idx').execute()
+  await db.schema.dropIndex('feed_item_originator_cursor_idx').execute()
+}
diff --git a/packages/pds/src/db/migrations/index.ts b/packages/pds/src/db/migrations/index.ts
@@ -53,3 +53,4 @@ export * as _20230523T183902064Z from './20230523T183902064Z-algo-whats-hot-view
 export * as _20230529T222706121Z from './20230529T222706121Z-suggested-follows'
 export * as _20230530T213530067Z from './20230530T213530067Z-rebase-indices'
 export * as _20230605T235529700Z from './20230605T235529700Z-outgoing-repo-seq'
+export * as _20230703T044601833Z from './20230703T044601833Z-feed-and-label-indices'