From 7637fdbf10eb2bdfe916e24236de7b5c363f659f Mon Sep 17 00:00:00 2001 From: devin ivy Date: Mon, 3 Jul 2023 09:48:27 -0400 Subject: [PATCH 01/13] backpressure on bsky backfill indexing (#1268) * backpressure on bsky backfill indexing * skip image resolution for text labeler * increase background queue concurrency for backfill * tidy --- packages/bsky/src/background.ts | 2 +- packages/bsky/src/config.ts | 12 ++++++++++++ packages/bsky/src/index.ts | 7 ++++++- packages/bsky/src/labeler/keyword.ts | 10 +++++++++- packages/bsky/src/subscription/repo.ts | 10 +++++++--- packages/bsky/src/subscription/util.ts | 6 +++++- 6 files changed, 40 insertions(+), 7 deletions(-) diff --git a/packages/bsky/src/background.ts b/packages/bsky/src/background.ts index a66ecf887b8..dbf47eb8868 100644 --- a/packages/bsky/src/background.ts +++ b/packages/bsky/src/background.ts @@ -5,7 +5,7 @@ import { dbLogger } from './logger' // A simple queue for in-process, out-of-band/backgrounded work export class BackgroundQueue { - queue = new PQueue({ concurrency: 10 }) + queue = new PQueue({ concurrency: 20 }) destroyed = false constructor(public db: Database) {} diff --git a/packages/bsky/src/config.ts b/packages/bsky/src/config.ts index de88cbe98a7..6d14771731b 100644 --- a/packages/bsky/src/config.ts +++ b/packages/bsky/src/config.ts @@ -23,6 +23,7 @@ export interface ServerConfigValues { hiveApiKey?: string adminPassword: string labelerKeywords: Record + indexerConcurrency?: number } export class ServerConfig { @@ -61,6 +62,7 @@ export class ServerConfig { const adminPassword = process.env.ADMIN_PASSWORD || 'admin' const labelerDid = process.env.LABELER_DID || 'did:example:labeler' const hiveApiKey = process.env.HIVE_API_KEY || undefined + const indexerConcurrency = maybeParseInt(process.env.INDEXER_CONCURRENCY) const labelerKeywords = {} return new ServerConfig({ version, @@ -83,6 +85,7 @@ export class ServerConfig { hiveApiKey, adminPassword, labelerKeywords, + indexerConcurrency, ...stripUndefineds(overrides ?? {}), }) } @@ -183,6 +186,10 @@ export class ServerConfig { get adminPassword() { return this.cfg.adminPassword } + + get indexerConcurrency() { + return this.cfg.indexerConcurrency + } } function stripUndefineds( @@ -196,3 +203,8 @@ function stripUndefineds( }) return result } + +function maybeParseInt(str) { + const parsed = parseInt(str) + return isNaN(parsed) ? undefined : parsed +} diff --git a/packages/bsky/src/index.ts b/packages/bsky/src/index.ts index 708f9e463d4..771b0ef27c3 100644 --- a/packages/bsky/src/index.ts +++ b/packages/bsky/src/index.ts @@ -157,7 +157,12 @@ export class BskyAppView { app.use(error.handler) const sub = config.repoProvider - ? new RepoSubscription(ctx, config.repoProvider, config.repoSubLockId) + ? new RepoSubscription( + ctx, + config.repoProvider, + config.repoSubLockId, + config.indexerConcurrency, + ) : undefined return new BskyAppView({ ctx, app, sub }) diff --git a/packages/bsky/src/labeler/keyword.ts b/packages/bsky/src/labeler/keyword.ts index d4d0e7735ac..b3fd745eaf1 100644 --- a/packages/bsky/src/labeler/keyword.ts +++ b/packages/bsky/src/labeler/keyword.ts @@ -1,9 +1,10 @@ import Database from '../db' import { Labeler } from './base' -import { keywordLabeling } from './util' +import { getFieldsFromRecord, keywordLabeling } from './util' import { IdResolver } from '@atproto/identity' import { ServerConfig } from '../config' import { BackgroundQueue } from '../background' +import { AtUri } from '@atproto/uri' export class KeywordLabeler extends Labeler { keywords: Record @@ -20,6 +21,13 @@ export class KeywordLabeler extends Labeler { this.keywords = ctx.cfg.labelerKeywords } + async labelRecord(_uri: AtUri, obj: unknown): Promise { + // skip image resolution + const { text } = getFieldsFromRecord(obj) + const txtLabels = await this.labelText(text.join(' ')) + return txtLabels + } + async labelText(text: string): Promise { return keywordLabeling(this.keywords, text) } diff --git a/packages/bsky/src/subscription/repo.ts b/packages/bsky/src/subscription/repo.ts index 174bd283d0f..2f4047805d1 100644 --- a/packages/bsky/src/subscription/repo.ts +++ b/packages/bsky/src/subscription/repo.ts @@ -25,7 +25,7 @@ export const REPO_SUB_ID = 1000 export class RepoSubscription { leader = new Leader(this.subLockId, this.ctx.db) - repoQueue = new PartitionedQueue() + repoQueue: PartitionedQueue cursorQueue = new LatestQueue() consecutive = new ConsecutiveList() destroyed = false @@ -34,7 +34,10 @@ export class RepoSubscription { public ctx: AppContext, public service: string, public subLockId = REPO_SUB_ID, - ) {} + public concurrency = Infinity, + ) { + this.repoQueue = new PartitionedQueue({ concurrency }) + } async run() { while (!this.destroyed) { @@ -81,6 +84,7 @@ export class RepoSubscription { ) }) }) + await this.repoQueue.main.onEmpty() // backpressure } }) if (ran && !this.destroyed) { @@ -107,7 +111,7 @@ export class RepoSubscription { async resume() { this.destroyed = false - this.repoQueue = new PartitionedQueue() + this.repoQueue = new PartitionedQueue({ concurrency: this.concurrency }) this.cursorQueue = new LatestQueue() this.consecutive = new ConsecutiveList() await this.run() diff --git a/packages/bsky/src/subscription/util.ts b/packages/bsky/src/subscription/util.ts index 70f3ec74905..0bd6b862176 100644 --- a/packages/bsky/src/subscription/util.ts +++ b/packages/bsky/src/subscription/util.ts @@ -3,9 +3,13 @@ import PQueue from 'p-queue' // A queue with arbitrarily many partitions, each processing work sequentially. // Partitions are created lazily and taken out of memory when they go idle. export class PartitionedQueue { - main = new PQueue({ concurrency: Infinity }) + main: PQueue partitions = new Map() + constructor(opts: { concurrency: number }) { + this.main = new PQueue({ concurrency: opts.concurrency }) + } + async add(partitionId: string, task: () => Promise) { if (this.main.isPaused) return return this.main.add(() => { From e7a0d27f1fef15d68a04be81cec449bfe3b1011f Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Mon, 3 Jul 2023 11:28:39 -0500 Subject: [PATCH 02/13] Proxy timeline skeleton construction (#1264) proxy timeline skeleton construction to appview --- .../bsky/unspecced/getTimelineSkeleton.json | 34 + packages/api/src/client/index.ts | 13 + packages/api/src/client/lexicons.ts | 49 + .../app/bsky/unspecced/getTimelineSkeleton.ts | 45 + .../getPopularFeedGenerators.ts} | 6 +- .../app/bsky/unspecced/getTimelineSkeleton.ts | 80 ++ packages/bsky/src/api/index.ts | 6 +- packages/bsky/src/lexicon/index.ts | 11 + packages/bsky/src/lexicon/lexicons.ts | 49 + .../app/bsky/unspecced/getTimelineSkeleton.ts | 46 + packages/dev-env/src/pds.ts | 2 +- packages/dev-env/src/types.ts | 3 +- .../src/app-view/api/app/bsky/feed/getFeed.ts | 18 +- .../app-view/api/app/bsky/feed/getTimeline.ts | 32 +- packages/pds/src/lexicon/index.ts | 11 + packages/pds/src/lexicon/lexicons.ts | 49 + .../app/bsky/unspecced/getTimelineSkeleton.ts | 46 + .../timeline-skeleton.test.ts.snap | 1227 +++++++++++++++++ .../tests/proxied/timeline-skeleton.test.ts | 59 + packages/pds/tests/proxied/views.test.ts | 4 +- 20 files changed, 1768 insertions(+), 22 deletions(-) create mode 100644 lexicons/app/bsky/unspecced/getTimelineSkeleton.json create mode 100644 packages/api/src/client/types/app/bsky/unspecced/getTimelineSkeleton.ts rename packages/bsky/src/api/app/bsky/{unspecced.ts => unspecced/getPopularFeedGenerators.ts} (92%) create mode 100644 packages/bsky/src/api/app/bsky/unspecced/getTimelineSkeleton.ts create mode 100644 packages/bsky/src/lexicon/types/app/bsky/unspecced/getTimelineSkeleton.ts create mode 100644 packages/pds/src/lexicon/types/app/bsky/unspecced/getTimelineSkeleton.ts create mode 100644 packages/pds/tests/proxied/__snapshots__/timeline-skeleton.test.ts.snap create mode 100644 packages/pds/tests/proxied/timeline-skeleton.test.ts diff --git a/lexicons/app/bsky/unspecced/getTimelineSkeleton.json b/lexicons/app/bsky/unspecced/getTimelineSkeleton.json new file mode 100644 index 00000000000..7ac6d7dbdec --- /dev/null +++ b/lexicons/app/bsky/unspecced/getTimelineSkeleton.json @@ -0,0 +1,34 @@ +{ + "lexicon": 1, + "id": "app.bsky.unspecced.getTimelineSkeleton", + "defs": { + "main": { + "type": "query", + "description": "A skeleton of a timeline - UNSPECCED & WILL GO AWAY SOON", + "parameters": { + "type": "params", + "properties": { + "limit": {"type": "integer", "minimum": 1, "maximum": 100, "default": 50}, + "cursor": {"type": "string"} + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["feed"], + "properties": { + "cursor": {"type": "string"}, + "feed": { + "type": "array", + "items": {"type": "ref", "ref": "app.bsky.feed.defs#skeletonFeedPost"} + } + } + } + }, + "errors": [ + {"name": "UnknownFeed"} + ] + } + } +} diff --git a/packages/api/src/client/index.ts b/packages/api/src/client/index.ts index e74bee840e0..9e66af0441d 100644 --- a/packages/api/src/client/index.ts +++ b/packages/api/src/client/index.ts @@ -123,6 +123,7 @@ import * as AppBskyNotificationUpdateSeen from './types/app/bsky/notification/up import * as AppBskyRichtextFacet from './types/app/bsky/richtext/facet' import * as AppBskyUnspeccedGetPopular from './types/app/bsky/unspecced/getPopular' import * as AppBskyUnspeccedGetPopularFeedGenerators from './types/app/bsky/unspecced/getPopularFeedGenerators' +import * as AppBskyUnspeccedGetTimelineSkeleton from './types/app/bsky/unspecced/getTimelineSkeleton' export * as ComAtprotoAdminDefs from './types/com/atproto/admin/defs' export * as ComAtprotoAdminDisableAccountInvites from './types/com/atproto/admin/disableAccountInvites' @@ -240,6 +241,7 @@ export * as AppBskyNotificationUpdateSeen from './types/app/bsky/notification/up export * as AppBskyRichtextFacet from './types/app/bsky/richtext/facet' export * as AppBskyUnspeccedGetPopular from './types/app/bsky/unspecced/getPopular' export * as AppBskyUnspeccedGetPopularFeedGenerators from './types/app/bsky/unspecced/getPopularFeedGenerators' +export * as AppBskyUnspeccedGetTimelineSkeleton from './types/app/bsky/unspecced/getTimelineSkeleton' export const COM_ATPROTO_ADMIN = { DefsTakedown: 'com.atproto.admin.defs#takedown', @@ -2047,4 +2049,15 @@ export class UnspeccedNS { throw AppBskyUnspeccedGetPopularFeedGenerators.toKnownErr(e) }) } + + getTimelineSkeleton( + params?: AppBskyUnspeccedGetTimelineSkeleton.QueryParams, + opts?: AppBskyUnspeccedGetTimelineSkeleton.CallOptions, + ): Promise { + return this._service.xrpc + .call('app.bsky.unspecced.getTimelineSkeleton', params, undefined, opts) + .catch((e) => { + throw AppBskyUnspeccedGetTimelineSkeleton.toKnownErr(e) + }) + } } diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index e7a5195328e..f2adcb93c69 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -6304,6 +6304,54 @@ export const schemaDict = { }, }, }, + AppBskyUnspeccedGetTimelineSkeleton: { + lexicon: 1, + id: 'app.bsky.unspecced.getTimelineSkeleton', + defs: { + main: { + type: 'query', + description: 'A skeleton of a timeline', + parameters: { + type: 'params', + properties: { + limit: { + type: 'integer', + minimum: 1, + maximum: 100, + default: 50, + }, + cursor: { + type: 'string', + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['feed'], + properties: { + cursor: { + type: 'string', + }, + feed: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:app.bsky.feed.defs#skeletonFeedPost', + }, + }, + }, + }, + }, + errors: [ + { + name: 'UnknownFeed', + }, + ], + }, + }, + }, } export const schemas: LexiconDoc[] = Object.values(schemaDict) as LexiconDoc[] export const lexicons: Lexicons = new Lexicons(schemas) @@ -6432,4 +6480,5 @@ export const ids = { AppBskyUnspeccedGetPopular: 'app.bsky.unspecced.getPopular', AppBskyUnspeccedGetPopularFeedGenerators: 'app.bsky.unspecced.getPopularFeedGenerators', + AppBskyUnspeccedGetTimelineSkeleton: 'app.bsky.unspecced.getTimelineSkeleton', } diff --git a/packages/api/src/client/types/app/bsky/unspecced/getTimelineSkeleton.ts b/packages/api/src/client/types/app/bsky/unspecced/getTimelineSkeleton.ts new file mode 100644 index 00000000000..de9ba9c0ae9 --- /dev/null +++ b/packages/api/src/client/types/app/bsky/unspecced/getTimelineSkeleton.ts @@ -0,0 +1,45 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import { Headers, XRPCError } from '@atproto/xrpc' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { isObj, hasProp } from '../../../../util' +import { lexicons } from '../../../../lexicons' +import { CID } from 'multiformats/cid' +import * as AppBskyFeedDefs from '../feed/defs' + +export interface QueryParams { + limit?: number + cursor?: string +} + +export type InputSchema = undefined + +export interface OutputSchema { + cursor?: string + feed: AppBskyFeedDefs.SkeletonFeedPost[] + [k: string]: unknown +} + +export interface CallOptions { + headers?: Headers +} + +export interface Response { + success: boolean + headers: Headers + data: OutputSchema +} + +export class UnknownFeedError extends XRPCError { + constructor(src: XRPCError) { + super(src.status, src.error, src.message) + } +} + +export function toKnownErr(e: any) { + if (e instanceof XRPCError) { + if (e.error === 'UnknownFeed') return new UnknownFeedError(e) + } + return e +} diff --git a/packages/bsky/src/api/app/bsky/unspecced.ts b/packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts similarity index 92% rename from packages/bsky/src/api/app/bsky/unspecced.ts rename to packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts index 1f4e4bf7f41..303fed257cb 100644 --- a/packages/bsky/src/api/app/bsky/unspecced.ts +++ b/packages/bsky/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts @@ -1,6 +1,6 @@ -import { Server } from '../../../lexicon' -import AppContext from '../../../context' -import { countAll } from '../../../db/util' +import { Server } from '../../../../lexicon' +import AppContext from '../../../../context' +import { countAll } from '../../../../db/util' // THIS IS A TEMPORARY UNSPECCED ROUTE export default function (server: Server, ctx: AppContext) { diff --git a/packages/bsky/src/api/app/bsky/unspecced/getTimelineSkeleton.ts b/packages/bsky/src/api/app/bsky/unspecced/getTimelineSkeleton.ts new file mode 100644 index 00000000000..0684ce91144 --- /dev/null +++ b/packages/bsky/src/api/app/bsky/unspecced/getTimelineSkeleton.ts @@ -0,0 +1,80 @@ +import { Server } from '../../../../lexicon' +import AppContext from '../../../../context' +import { FeedKeyset, getFeedDateThreshold } from '../util/feed' +import { paginate } from '../../../../db/pagination' + +// THIS IS A TEMPORARY UNSPECCED ROUTE +export default function (server: Server, ctx: AppContext) { + server.app.bsky.unspecced.getTimelineSkeleton({ + auth: ctx.authVerifier, + handler: async ({ auth, params }) => { + const { limit, cursor } = params + const viewer = auth.credentials.did + + const db = ctx.db.db + const { ref } = db.dynamic + + const feedService = ctx.services.feed(ctx.db) + const graphService = ctx.services.graph(ctx.db) + + const followingIdsSubquery = db + .selectFrom('follow') + .select('follow.subjectDid') + .where('follow.creator', '=', viewer) + + const keyset = new FeedKeyset( + ref('feed_item.sortAt'), + ref('feed_item.cid'), + ) + const sortFrom = keyset.unpack(cursor)?.primary + + let feedItemsQb = feedService + .selectFeedItemQb() + .where((qb) => + qb + .where('originatorDid', '=', viewer) + .orWhere('originatorDid', 'in', followingIdsSubquery), + ) + .where((qb) => + // Hide posts and reposts of or by muted actors + graphService.whereNotMuted(qb, viewer, [ + ref('post.creator'), + ref('originatorDid'), + ]), + ) + .whereNotExists( + graphService.blockQb(viewer, [ + ref('post.creator'), + ref('originatorDid'), + ]), + ) + .where('feed_item.sortAt', '>', getFeedDateThreshold(sortFrom)) + + feedItemsQb = paginate(feedItemsQb, { + limit, + cursor, + keyset, + tryIndex: true, + }) + + const feedItems = await feedItemsQb.execute() + const feed = feedItems.map((item) => ({ + post: item.postUri, + reason: + item.uri === item.postUri + ? undefined + : { + $type: 'app.bsky.feed.defs#skeletonReasonRepost', + repost: item.uri, + }, + })) + return { + encoding: 'application/json', + body: { + cursor: keyset.packFromResult(feedItems), + feed, + }, + } + }, + }) +} diff --git a/packages/bsky/src/api/index.ts b/packages/bsky/src/api/index.ts index c56765c8d56..691eae882d1 100644 --- a/packages/bsky/src/api/index.ts +++ b/packages/bsky/src/api/index.ts @@ -30,7 +30,8 @@ import getSuggestions from './app/bsky/actor/getSuggestions' import getUnreadCount from './app/bsky/notification/getUnreadCount' import listNotifications from './app/bsky/notification/listNotifications' import updateSeen from './app/bsky/notification/updateSeen' -import unspecced from './app/bsky/unspecced' +import getPopularFeedGenerators from './app/bsky/unspecced/getPopularFeedGenerators' +import getTimelineSkeleton from './app/bsky/unspecced/getTimelineSkeleton' import createReport from './com/atproto/moderation/createReport' import resolveModerationReports from './com/atproto/admin/resolveModerationReports' import reverseModerationAction from './com/atproto/admin/reverseModerationAction' @@ -81,7 +82,8 @@ export default function (server: Server, ctx: AppContext) { getUnreadCount(server, ctx) listNotifications(server, ctx) updateSeen(server, ctx) - unspecced(server, ctx) + getPopularFeedGenerators(server, ctx) + getTimelineSkeleton(server, ctx) // com.atproto createReport(server, ctx) resolveModerationReports(server, ctx) diff --git a/packages/bsky/src/lexicon/index.ts b/packages/bsky/src/lexicon/index.ts index 548011fc331..982ed642924 100644 --- a/packages/bsky/src/lexicon/index.ts +++ b/packages/bsky/src/lexicon/index.ts @@ -103,6 +103,7 @@ import * as AppBskyNotificationListNotifications from './types/app/bsky/notifica import * as AppBskyNotificationUpdateSeen from './types/app/bsky/notification/updateSeen' import * as AppBskyUnspeccedGetPopular from './types/app/bsky/unspecced/getPopular' import * as AppBskyUnspeccedGetPopularFeedGenerators from './types/app/bsky/unspecced/getPopularFeedGenerators' +import * as AppBskyUnspeccedGetTimelineSkeleton from './types/app/bsky/unspecced/getTimelineSkeleton' export const COM_ATPROTO_ADMIN = { DefsTakedown: 'com.atproto.admin.defs#takedown', @@ -1048,6 +1049,16 @@ export class UnspeccedNS { const nsid = 'app.bsky.unspecced.getPopularFeedGenerators' // @ts-ignore return this._server.xrpc.method(nsid, cfg) } + + getTimelineSkeleton( + cfg: ConfigOf< + AV, + AppBskyUnspeccedGetTimelineSkeleton.Handler> + >, + ) { + const nsid = 'app.bsky.unspecced.getTimelineSkeleton' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } } type ConfigOf = diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index e7a5195328e..f2adcb93c69 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -6304,6 +6304,54 @@ export const schemaDict = { }, }, }, + AppBskyUnspeccedGetTimelineSkeleton: { + lexicon: 1, + id: 'app.bsky.unspecced.getTimelineSkeleton', + defs: { + main: { + type: 'query', + description: 'A skeleton of a timeline', + parameters: { + type: 'params', + properties: { + limit: { + type: 'integer', + minimum: 1, + maximum: 100, + default: 50, + }, + cursor: { + type: 'string', + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['feed'], + properties: { + cursor: { + type: 'string', + }, + feed: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:app.bsky.feed.defs#skeletonFeedPost', + }, + }, + }, + }, + }, + errors: [ + { + name: 'UnknownFeed', + }, + ], + }, + }, + }, } export const schemas: LexiconDoc[] = Object.values(schemaDict) as LexiconDoc[] export const lexicons: Lexicons = new Lexicons(schemas) @@ -6432,4 +6480,5 @@ export const ids = { AppBskyUnspeccedGetPopular: 'app.bsky.unspecced.getPopular', AppBskyUnspeccedGetPopularFeedGenerators: 'app.bsky.unspecced.getPopularFeedGenerators', + AppBskyUnspeccedGetTimelineSkeleton: 'app.bsky.unspecced.getTimelineSkeleton', } diff --git a/packages/bsky/src/lexicon/types/app/bsky/unspecced/getTimelineSkeleton.ts b/packages/bsky/src/lexicon/types/app/bsky/unspecced/getTimelineSkeleton.ts new file mode 100644 index 00000000000..1178a844a93 --- /dev/null +++ b/packages/bsky/src/lexicon/types/app/bsky/unspecced/getTimelineSkeleton.ts @@ -0,0 +1,46 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' +import * as AppBskyFeedDefs from '../feed/defs' + +export interface QueryParams { + limit: number + cursor?: string +} + +export type InputSchema = undefined + +export interface OutputSchema { + cursor?: string + feed: AppBskyFeedDefs.SkeletonFeedPost[] + [k: string]: unknown +} + +export type HandlerInput = undefined + +export interface HandlerSuccess { + encoding: 'application/json' + body: OutputSchema + headers?: { [key: string]: string } +} + +export interface HandlerError { + status: number + message?: string + error?: 'UnknownFeed' +} + +export type HandlerOutput = HandlerError | HandlerSuccess +export type Handler = (ctx: { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +}) => Promise | HandlerOutput diff --git a/packages/dev-env/src/pds.ts b/packages/dev-env/src/pds.ts index facb5ceed4b..6a818e4e235 100644 --- a/packages/dev-env/src/pds.ts +++ b/packages/dev-env/src/pds.ts @@ -77,7 +77,7 @@ export class TestPds { : pds.Database.memory() await db.migrateToLatestOrThrow() - if (config.bskyAppViewEndpoint) { + if (config.bskyAppViewEndpoint && !cfg.enableAppView) { // Disable communication to app view within pds MessageDispatcher.prototype.send = async () => {} } diff --git a/packages/dev-env/src/types.ts b/packages/dev-env/src/types.ts index ec088ec4806..408ce8cee4f 100644 --- a/packages/dev-env/src/types.ts +++ b/packages/dev-env/src/types.ts @@ -9,6 +9,7 @@ export type PlcConfig = { export type PdsConfig = Partial & { plcUrl: string migration?: string + enableAppView?: boolean } export type BskyConfig = Partial & { @@ -22,7 +23,7 @@ export type BskyConfig = Partial & { export type TestServerParams = { dbPostgresUrl: string dbPostgresSchema: string - pds: Partial + pds: Partial plc: Partial bsky: Partial } diff --git a/packages/pds/src/app-view/api/app/bsky/feed/getFeed.ts b/packages/pds/src/app-view/api/app/bsky/feed/getFeed.ts index ca1ea03c495..e521f4fda34 100644 --- a/packages/pds/src/app-view/api/app/bsky/feed/getFeed.ts +++ b/packages/pds/src/app-view/api/app/bsky/feed/getFeed.ts @@ -139,9 +139,17 @@ async function skeletonFromFeedGen( throw err } + return filterMutesAndBlocks(ctx, skeleton, params.limit, requester) +} + +export async function filterMutesAndBlocks( + ctx: AppContext, + skeleton: SkeletonOutput, + limit: number, + requester: string, +) { const { feed: skeletonFeed, ...rest } = skeleton - // Hydrate feed skeleton const { ref } = ctx.db.db.dynamic const feedService = ctx.services.appView.feed(ctx.db) const graphService = ctx.services.appView.graph(ctx.db) @@ -168,7 +176,7 @@ async function skeletonFromFeedGen( .execute() : [] - const orderedItems = getOrderedFeedItems(skeletonFeed, feedItems, params) + const orderedItems = getOrderedFeedItems(skeletonFeed, feedItems, limit) return { ...rest, feedItems: orderedItems, @@ -185,15 +193,15 @@ function getSkeleFeedItemUri(item: SkeletonFeedPost) { function getOrderedFeedItems( skeletonItems: SkeletonFeedPost[], feedItems: FeedRow[], - params: GetFeedParams, + limit: number, ) { const SKIP = [] const feedItemsByUri = feedItems.reduce((acc, item) => { return Object.assign(acc, { [item.uri]: item }) }, {} as Record) // enforce limit param in the case that the feedgen does not - if (skeletonItems.length > params.limit) { - skeletonItems = skeletonItems.slice(0, params.limit) + if (skeletonItems.length > limit) { + skeletonItems = skeletonItems.slice(0, limit) } return skeletonItems.flatMap((item) => { const uri = getSkeleFeedItemUri(item) diff --git a/packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts b/packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts index 8da47aab4af..c6b0bd9e7a4 100644 --- a/packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts +++ b/packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts @@ -4,31 +4,45 @@ import { FeedAlgorithm, FeedKeyset, getFeedDateThreshold } from '../util/feed' import { paginate } from '../../../../../db/pagination' import AppContext from '../../../../../context' import { FeedRow } from '../../../../services/feed' +import { filterMutesAndBlocks } from './getFeed' export default function (server: Server, ctx: AppContext) { server.app.bsky.feed.getTimeline({ auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did + const { algorithm, limit, cursor } = params + if (algorithm && algorithm !== FeedAlgorithm.ReverseChronological) { + throw new InvalidRequestError(`Unsupported algorithm: ${algorithm}`) + } + if (ctx.canProxy(req)) { - const res = await ctx.appviewAgent.api.app.bsky.feed.getTimeline( - params, - await ctx.serviceAuthHeaders(requester), + const res = + await ctx.appviewAgent.api.app.bsky.unspecced.getTimelineSkeleton( + params, + await ctx.serviceAuthHeaders(requester), + ) + const filtered = await filterMutesAndBlocks( + ctx, + res.data, + limit, + requester, ) + const hydrated = await ctx.services.appView + .feed(ctx.db) + .hydrateFeed(filtered.feedItems, requester) return { encoding: 'application/json', - body: res.data, + body: { + cursor: filtered.cursor, + feed: hydrated, + }, } } - const { algorithm, limit, cursor } = params const db = ctx.db.db const { ref } = db.dynamic - if (algorithm && algorithm !== FeedAlgorithm.ReverseChronological) { - throw new InvalidRequestError(`Unsupported algorithm: ${algorithm}`) - } - const accountService = ctx.services.account(ctx.db) const feedService = ctx.services.appView.feed(ctx.db) const graphService = ctx.services.appView.graph(ctx.db) diff --git a/packages/pds/src/lexicon/index.ts b/packages/pds/src/lexicon/index.ts index 548011fc331..982ed642924 100644 --- a/packages/pds/src/lexicon/index.ts +++ b/packages/pds/src/lexicon/index.ts @@ -103,6 +103,7 @@ import * as AppBskyNotificationListNotifications from './types/app/bsky/notifica import * as AppBskyNotificationUpdateSeen from './types/app/bsky/notification/updateSeen' import * as AppBskyUnspeccedGetPopular from './types/app/bsky/unspecced/getPopular' import * as AppBskyUnspeccedGetPopularFeedGenerators from './types/app/bsky/unspecced/getPopularFeedGenerators' +import * as AppBskyUnspeccedGetTimelineSkeleton from './types/app/bsky/unspecced/getTimelineSkeleton' export const COM_ATPROTO_ADMIN = { DefsTakedown: 'com.atproto.admin.defs#takedown', @@ -1048,6 +1049,16 @@ export class UnspeccedNS { const nsid = 'app.bsky.unspecced.getPopularFeedGenerators' // @ts-ignore return this._server.xrpc.method(nsid, cfg) } + + getTimelineSkeleton( + cfg: ConfigOf< + AV, + AppBskyUnspeccedGetTimelineSkeleton.Handler> + >, + ) { + const nsid = 'app.bsky.unspecced.getTimelineSkeleton' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } } type ConfigOf = diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index e7a5195328e..f2adcb93c69 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -6304,6 +6304,54 @@ export const schemaDict = { }, }, }, + AppBskyUnspeccedGetTimelineSkeleton: { + lexicon: 1, + id: 'app.bsky.unspecced.getTimelineSkeleton', + defs: { + main: { + type: 'query', + description: 'A skeleton of a timeline', + parameters: { + type: 'params', + properties: { + limit: { + type: 'integer', + minimum: 1, + maximum: 100, + default: 50, + }, + cursor: { + type: 'string', + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['feed'], + properties: { + cursor: { + type: 'string', + }, + feed: { + type: 'array', + items: { + type: 'ref', + ref: 'lex:app.bsky.feed.defs#skeletonFeedPost', + }, + }, + }, + }, + }, + errors: [ + { + name: 'UnknownFeed', + }, + ], + }, + }, + }, } export const schemas: LexiconDoc[] = Object.values(schemaDict) as LexiconDoc[] export const lexicons: Lexicons = new Lexicons(schemas) @@ -6432,4 +6480,5 @@ export const ids = { AppBskyUnspeccedGetPopular: 'app.bsky.unspecced.getPopular', AppBskyUnspeccedGetPopularFeedGenerators: 'app.bsky.unspecced.getPopularFeedGenerators', + AppBskyUnspeccedGetTimelineSkeleton: 'app.bsky.unspecced.getTimelineSkeleton', } diff --git a/packages/pds/src/lexicon/types/app/bsky/unspecced/getTimelineSkeleton.ts b/packages/pds/src/lexicon/types/app/bsky/unspecced/getTimelineSkeleton.ts new file mode 100644 index 00000000000..1178a844a93 --- /dev/null +++ b/packages/pds/src/lexicon/types/app/bsky/unspecced/getTimelineSkeleton.ts @@ -0,0 +1,46 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import { ValidationResult, BlobRef } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { CID } from 'multiformats/cid' +import { HandlerAuth } from '@atproto/xrpc-server' +import * as AppBskyFeedDefs from '../feed/defs' + +export interface QueryParams { + limit: number + cursor?: string +} + +export type InputSchema = undefined + +export interface OutputSchema { + cursor?: string + feed: AppBskyFeedDefs.SkeletonFeedPost[] + [k: string]: unknown +} + +export type HandlerInput = undefined + +export interface HandlerSuccess { + encoding: 'application/json' + body: OutputSchema + headers?: { [key: string]: string } +} + +export interface HandlerError { + status: number + message?: string + error?: 'UnknownFeed' +} + +export type HandlerOutput = HandlerError | HandlerSuccess +export type Handler = (ctx: { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +}) => Promise | HandlerOutput diff --git a/packages/pds/tests/proxied/__snapshots__/timeline-skeleton.test.ts.snap b/packages/pds/tests/proxied/__snapshots__/timeline-skeleton.test.ts.snap new file mode 100644 index 00000000000..3c6ca0ad290 --- /dev/null +++ b/packages/pds/tests/proxied/__snapshots__/timeline-skeleton.test.ts.snap @@ -0,0 +1,1227 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`proxies timeline skeleton timeline skeleton construction 1`] = ` +Object { + "cursor": "0000000000000::bafycid", + "feed": Array [ + Object { + "post": Object { + "author": Object { + "avatar": "https://pds.public.url/image/KzkHFsMRQ6oAKCHCRKFA1H-rDdc7VOtvEVpUJ82TwyQ/rs:fill:1000:1000:1:0/plain/bafkreiaivizp4xldojmmpuzmiu75cmea7nq56dnntnuhzhsjcb63aou5ei@jpeg", + "did": "user(0)", + "displayName": "ali", + "handle": "alice.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "muted": false, + }, + }, + "cid": "cids(0)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 3, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000000Z", + "text": "again", + }, + "replyCount": 2, + "repostCount": 1, + "uri": "record(0)", + "viewer": Object {}, + }, + "reason": Object { + "$type": "app.bsky.feed.defs#reasonRepost", + "by": Object { + "did": "user(1)", + "handle": "dan.test", + "labels": Array [ + Object { + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "did:example:labeler", + "uri": "user(1)", + "val": "repo-action-label", + }, + ], + "viewer": Object { + "blockedBy": false, + "following": "record(1)", + "muted": false, + }, + }, + "indexedAt": "1970-01-01T00:00:00.000Z", + }, + }, + Object { + "post": Object { + "author": Object { + "did": "user(1)", + "handle": "dan.test", + "labels": Array [ + Object { + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "did:example:labeler", + "uri": "user(1)", + "val": "repo-action-label", + }, + ], + "viewer": Object { + "blockedBy": false, + "following": "record(1)", + "muted": false, + }, + }, + "cid": "cids(1)", + "embed": Object { + "$type": "app.bsky.embed.record#view", + "record": Object { + "$type": "app.bsky.embed.record#viewRecord", + "author": Object { + "did": "user(2)", + "handle": "carol.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "followedBy": "record(5)", + "following": "record(4)", + "muted": false, + }, + }, + "cid": "cids(2)", + "embeds": Array [ + Object { + "$type": "app.bsky.embed.recordWithMedia#view", + "media": Object { + "$type": "app.bsky.embed.images#view", + "images": Array [ + Object { + "alt": "tests/image/fixtures/key-landscape-small.jpg", + "fullsize": "https://pds.public.url/image/AiDXkxVbgBksxb1nfiRn1m6S4K8_mee6o8r-UGLNzOM/rs:fit:2000:2000:1:0/plain/bafkreigy5p3xxceipk2o6nqtnugpft26ol6yleqhboqziino7axvdngtci@jpeg", + "thumb": "https://pds.public.url/image/uc7FGfiGv0mMqmk9XiqHXrIhNymLHaex7Ge8nEhmXqo/rs:fit:1000:1000:1:0/plain/bafkreigy5p3xxceipk2o6nqtnugpft26ol6yleqhboqziino7axvdngtci@jpeg", + }, + Object { + "alt": "tests/image/fixtures/key-alt.jpg", + "fullsize": "https://pds.public.url/image/xC2No-8rKVDIwIMmCiEBm9EiGLDBBOpf36PHoGf-GDw/rs:fit:2000:2000:1:0/plain/bafkreifdklbbcdsyanjz3oqe5pf2omuq5ansthokxlbleagg3eenx62h7e@jpeg", + "thumb": "https://pds.public.url/image/g7yazUpNwN8LKumZ2Zmn_ptQbtMLs1Pti5-GDn7H8_8/rs:fit:1000:1000:1:0/plain/bafkreifdklbbcdsyanjz3oqe5pf2omuq5ansthokxlbleagg3eenx62h7e@jpeg", + }, + ], + }, + "record": Object { + "record": Object { + "$type": "app.bsky.embed.record#viewRecord", + "author": Object { + "avatar": "https://pds.public.url/image/KzkHFsMRQ6oAKCHCRKFA1H-rDdc7VOtvEVpUJ82TwyQ/rs:fill:1000:1000:1:0/plain/bafkreiaivizp4xldojmmpuzmiu75cmea7nq56dnntnuhzhsjcb63aou5ei@jpeg", + "did": "user(3)", + "displayName": "bobby", + "handle": "bob.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "followedBy": "record(8)", + "following": "record(7)", + "muted": false, + }, + }, + "cid": "cids(5)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "uri": "record(6)", + "value": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "langs": Array [ + "en-US", + "i-klingon", + ], + "text": "bob back at it again!", + }, + }, + }, + }, + ], + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "uri": "record(3)", + "value": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "embed": Object { + "$type": "app.bsky.embed.recordWithMedia", + "media": Object { + "$type": "app.bsky.embed.images", + "images": Array [ + Object { + "alt": "tests/image/fixtures/key-landscape-small.jpg", + "image": Object { + "$type": "blob", + "mimeType": "image/jpeg", + "ref": Object { + "$link": "cids(3)", + }, + "size": 4114, + }, + }, + Object { + "alt": "tests/image/fixtures/key-alt.jpg", + "image": Object { + "$type": "blob", + "mimeType": "image/jpeg", + "ref": Object { + "$link": "cids(4)", + }, + "size": 12736, + }, + }, + ], + }, + "record": Object { + "record": Object { + "cid": "cids(5)", + "uri": "record(6)", + }, + }, + }, + "text": "hi im carol", + }, + }, + }, + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 0, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "embed": Object { + "$type": "app.bsky.embed.record", + "record": Object { + "cid": "cids(2)", + "uri": "record(3)", + }, + }, + "facets": Array [ + Object { + "features": Array [ + Object { + "$type": "app.bsky.richtext.facet#mention", + "did": "user(0)", + }, + ], + "index": Object { + "byteEnd": 18, + "byteStart": 0, + }, + }, + ], + "text": "@alice.bluesky.xyz is the best", + }, + "replyCount": 0, + "repostCount": 1, + "uri": "record(2)", + "viewer": Object {}, + }, + "reason": Object { + "$type": "app.bsky.feed.defs#reasonRepost", + "by": Object { + "did": "user(2)", + "handle": "carol.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "followedBy": "record(5)", + "following": "record(4)", + "muted": false, + }, + }, + "indexedAt": "1970-01-01T00:00:00.000Z", + }, + }, + Object { + "post": Object { + "author": Object { + "avatar": "https://pds.public.url/image/KzkHFsMRQ6oAKCHCRKFA1H-rDdc7VOtvEVpUJ82TwyQ/rs:fill:1000:1000:1:0/plain/bafkreiaivizp4xldojmmpuzmiu75cmea7nq56dnntnuhzhsjcb63aou5ei@jpeg", + "did": "user(0)", + "displayName": "ali", + "handle": "alice.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "muted": false, + }, + }, + "cid": "cids(6)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 0, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "reply": Object { + "parent": Object { + "cid": "cids(7)", + "uri": "record(10)", + }, + "root": Object { + "cid": "cids(0)", + "uri": "record(0)", + }, + }, + "text": "thanks bob", + }, + "replyCount": 0, + "repostCount": 0, + "uri": "record(9)", + "viewer": Object {}, + }, + "reply": Object { + "parent": Object { + "$type": "app.bsky.feed.defs#postView", + "author": Object { + "avatar": "https://pds.public.url/image/KzkHFsMRQ6oAKCHCRKFA1H-rDdc7VOtvEVpUJ82TwyQ/rs:fill:1000:1000:1:0/plain/bafkreiaivizp4xldojmmpuzmiu75cmea7nq56dnntnuhzhsjcb63aou5ei@jpeg", + "did": "user(3)", + "displayName": "bobby", + "handle": "bob.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "followedBy": "record(8)", + "following": "record(7)", + "muted": false, + }, + }, + "cid": "cids(7)", + "embed": Object { + "$type": "app.bsky.embed.images#view", + "images": Array [ + Object { + "alt": "tests/image/fixtures/key-landscape-small.jpg", + "fullsize": "https://pds.public.url/image/AiDXkxVbgBksxb1nfiRn1m6S4K8_mee6o8r-UGLNzOM/rs:fit:2000:2000:1:0/plain/bafkreigy5p3xxceipk2o6nqtnugpft26ol6yleqhboqziino7axvdngtci@jpeg", + "thumb": "https://pds.public.url/image/uc7FGfiGv0mMqmk9XiqHXrIhNymLHaex7Ge8nEhmXqo/rs:fit:1000:1000:1:0/plain/bafkreigy5p3xxceipk2o6nqtnugpft26ol6yleqhboqziino7axvdngtci@jpeg", + }, + ], + }, + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [ + Object { + "cid": "cids(7)", + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "did:example:labeler", + "uri": "record(10)", + "val": "test-label", + }, + Object { + "cid": "cids(7)", + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "did:example:labeler", + "uri": "record(10)", + "val": "test-label-2", + }, + ], + "likeCount": 0, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "embed": Object { + "$type": "app.bsky.embed.images", + "images": Array [ + Object { + "alt": "tests/image/fixtures/key-landscape-small.jpg", + "image": Object { + "$type": "blob", + "mimeType": "image/jpeg", + "ref": Object { + "$link": "cids(3)", + }, + "size": 4114, + }, + }, + ], + }, + "reply": Object { + "parent": Object { + "cid": "cids(0)", + "uri": "record(0)", + }, + "root": Object { + "cid": "cids(0)", + "uri": "record(0)", + }, + }, + "text": "hear that label_me label_me_2", + }, + "replyCount": 1, + "repostCount": 0, + "uri": "record(10)", + "viewer": Object {}, + }, + "root": Object { + "$type": "app.bsky.feed.defs#postView", + "author": Object { + "avatar": "https://pds.public.url/image/KzkHFsMRQ6oAKCHCRKFA1H-rDdc7VOtvEVpUJ82TwyQ/rs:fill:1000:1000:1:0/plain/bafkreiaivizp4xldojmmpuzmiu75cmea7nq56dnntnuhzhsjcb63aou5ei@jpeg", + "did": "user(0)", + "displayName": "ali", + "handle": "alice.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "muted": false, + }, + }, + "cid": "cids(0)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 3, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000000Z", + "text": "again", + }, + "replyCount": 2, + "repostCount": 1, + "uri": "record(0)", + "viewer": Object {}, + }, + }, + }, + Object { + "post": Object { + "author": Object { + "did": "user(2)", + "handle": "carol.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "followedBy": "record(5)", + "following": "record(4)", + "muted": false, + }, + }, + "cid": "cids(8)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 0, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "reply": Object { + "parent": Object { + "cid": "cids(0)", + "uri": "record(0)", + }, + "root": Object { + "cid": "cids(0)", + "uri": "record(0)", + }, + }, + "text": "of course", + }, + "replyCount": 0, + "repostCount": 0, + "uri": "record(11)", + "viewer": Object {}, + }, + "reply": Object { + "parent": Object { + "$type": "app.bsky.feed.defs#postView", + "author": Object { + "avatar": "https://pds.public.url/image/KzkHFsMRQ6oAKCHCRKFA1H-rDdc7VOtvEVpUJ82TwyQ/rs:fill:1000:1000:1:0/plain/bafkreiaivizp4xldojmmpuzmiu75cmea7nq56dnntnuhzhsjcb63aou5ei@jpeg", + "did": "user(0)", + "displayName": "ali", + "handle": "alice.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "muted": false, + }, + }, + "cid": "cids(0)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 3, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000000Z", + "text": "again", + }, + "replyCount": 2, + "repostCount": 1, + "uri": "record(0)", + "viewer": Object {}, + }, + "root": Object { + "$type": "app.bsky.feed.defs#postView", + "author": Object { + "avatar": "https://pds.public.url/image/KzkHFsMRQ6oAKCHCRKFA1H-rDdc7VOtvEVpUJ82TwyQ/rs:fill:1000:1000:1:0/plain/bafkreiaivizp4xldojmmpuzmiu75cmea7nq56dnntnuhzhsjcb63aou5ei@jpeg", + "did": "user(0)", + "displayName": "ali", + "handle": "alice.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "muted": false, + }, + }, + "cid": "cids(0)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 3, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000000Z", + "text": "again", + }, + "replyCount": 2, + "repostCount": 1, + "uri": "record(0)", + "viewer": Object {}, + }, + }, + }, + Object { + "post": Object { + "author": Object { + "avatar": "https://pds.public.url/image/KzkHFsMRQ6oAKCHCRKFA1H-rDdc7VOtvEVpUJ82TwyQ/rs:fill:1000:1000:1:0/plain/bafkreiaivizp4xldojmmpuzmiu75cmea7nq56dnntnuhzhsjcb63aou5ei@jpeg", + "did": "user(3)", + "displayName": "bobby", + "handle": "bob.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "followedBy": "record(8)", + "following": "record(7)", + "muted": false, + }, + }, + "cid": "cids(7)", + "embed": Object { + "$type": "app.bsky.embed.images#view", + "images": Array [ + Object { + "alt": "tests/image/fixtures/key-landscape-small.jpg", + "fullsize": "https://pds.public.url/image/AiDXkxVbgBksxb1nfiRn1m6S4K8_mee6o8r-UGLNzOM/rs:fit:2000:2000:1:0/plain/bafkreigy5p3xxceipk2o6nqtnugpft26ol6yleqhboqziino7axvdngtci@jpeg", + "thumb": "https://pds.public.url/image/uc7FGfiGv0mMqmk9XiqHXrIhNymLHaex7Ge8nEhmXqo/rs:fit:1000:1000:1:0/plain/bafkreigy5p3xxceipk2o6nqtnugpft26ol6yleqhboqziino7axvdngtci@jpeg", + }, + ], + }, + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [ + Object { + "cid": "cids(7)", + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "did:example:labeler", + "uri": "record(10)", + "val": "test-label", + }, + Object { + "cid": "cids(7)", + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "did:example:labeler", + "uri": "record(10)", + "val": "test-label-2", + }, + ], + "likeCount": 0, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "embed": Object { + "$type": "app.bsky.embed.images", + "images": Array [ + Object { + "alt": "tests/image/fixtures/key-landscape-small.jpg", + "image": Object { + "$type": "blob", + "mimeType": "image/jpeg", + "ref": Object { + "$link": "cids(3)", + }, + "size": 4114, + }, + }, + ], + }, + "reply": Object { + "parent": Object { + "cid": "cids(0)", + "uri": "record(0)", + }, + "root": Object { + "cid": "cids(0)", + "uri": "record(0)", + }, + }, + "text": "hear that label_me label_me_2", + }, + "replyCount": 1, + "repostCount": 0, + "uri": "record(10)", + "viewer": Object {}, + }, + "reply": Object { + "parent": Object { + "$type": "app.bsky.feed.defs#postView", + "author": Object { + "avatar": "https://pds.public.url/image/KzkHFsMRQ6oAKCHCRKFA1H-rDdc7VOtvEVpUJ82TwyQ/rs:fill:1000:1000:1:0/plain/bafkreiaivizp4xldojmmpuzmiu75cmea7nq56dnntnuhzhsjcb63aou5ei@jpeg", + "did": "user(0)", + "displayName": "ali", + "handle": "alice.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "muted": false, + }, + }, + "cid": "cids(0)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 3, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000000Z", + "text": "again", + }, + "replyCount": 2, + "repostCount": 1, + "uri": "record(0)", + "viewer": Object {}, + }, + "root": Object { + "$type": "app.bsky.feed.defs#postView", + "author": Object { + "avatar": "https://pds.public.url/image/KzkHFsMRQ6oAKCHCRKFA1H-rDdc7VOtvEVpUJ82TwyQ/rs:fill:1000:1000:1:0/plain/bafkreiaivizp4xldojmmpuzmiu75cmea7nq56dnntnuhzhsjcb63aou5ei@jpeg", + "did": "user(0)", + "displayName": "ali", + "handle": "alice.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "muted": false, + }, + }, + "cid": "cids(0)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 3, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000000Z", + "text": "again", + }, + "replyCount": 2, + "repostCount": 1, + "uri": "record(0)", + "viewer": Object {}, + }, + }, + }, + Object { + "post": Object { + "author": Object { + "avatar": "https://pds.public.url/image/KzkHFsMRQ6oAKCHCRKFA1H-rDdc7VOtvEVpUJ82TwyQ/rs:fill:1000:1000:1:0/plain/bafkreiaivizp4xldojmmpuzmiu75cmea7nq56dnntnuhzhsjcb63aou5ei@jpeg", + "did": "user(0)", + "displayName": "ali", + "handle": "alice.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "muted": false, + }, + }, + "cid": "cids(9)", + "embed": Object { + "$type": "app.bsky.embed.record#view", + "record": Object { + "$type": "app.bsky.embed.record#viewRecord", + "author": Object { + "did": "user(1)", + "handle": "dan.test", + "labels": Array [ + Object { + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "did:example:labeler", + "uri": "user(1)", + "val": "repo-action-label", + }, + ], + "viewer": Object { + "blockedBy": false, + "following": "record(1)", + "muted": false, + }, + }, + "cid": "cids(1)", + "embeds": Array [ + Object { + "$type": "app.bsky.embed.record#view", + "record": Object { + "$type": "app.bsky.embed.record#viewRecord", + "author": Object { + "did": "user(2)", + "handle": "carol.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "followedBy": "record(5)", + "following": "record(4)", + "muted": false, + }, + }, + "cid": "cids(2)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "uri": "record(3)", + "value": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "embed": Object { + "$type": "app.bsky.embed.recordWithMedia", + "media": Object { + "$type": "app.bsky.embed.images", + "images": Array [ + Object { + "alt": "tests/image/fixtures/key-landscape-small.jpg", + "image": Object { + "$type": "blob", + "mimeType": "image/jpeg", + "ref": Object { + "$link": "cids(3)", + }, + "size": 4114, + }, + }, + Object { + "alt": "tests/image/fixtures/key-alt.jpg", + "image": Object { + "$type": "blob", + "mimeType": "image/jpeg", + "ref": Object { + "$link": "cids(4)", + }, + "size": 12736, + }, + }, + ], + }, + "record": Object { + "record": Object { + "cid": "cids(5)", + "uri": "record(6)", + }, + }, + }, + "text": "hi im carol", + }, + }, + }, + ], + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "uri": "record(2)", + "value": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "embed": Object { + "$type": "app.bsky.embed.record", + "record": Object { + "cid": "cids(2)", + "uri": "record(3)", + }, + }, + "facets": Array [ + Object { + "features": Array [ + Object { + "$type": "app.bsky.richtext.facet#mention", + "did": "user(0)", + }, + ], + "index": Object { + "byteEnd": 18, + "byteStart": 0, + }, + }, + ], + "text": "@alice.bluesky.xyz is the best", + }, + }, + }, + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [ + Object { + "cid": "cids(9)", + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "did:example:labeler", + "uri": "record(12)", + "val": "test-label", + }, + ], + "likeCount": 2, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "embed": Object { + "$type": "app.bsky.embed.record", + "record": Object { + "cid": "cids(1)", + "uri": "record(2)", + }, + }, + "text": "yoohoo label_me", + }, + "replyCount": 0, + "repostCount": 0, + "uri": "record(12)", + "viewer": Object {}, + }, + }, + Object { + "post": Object { + "author": Object { + "avatar": "https://pds.public.url/image/KzkHFsMRQ6oAKCHCRKFA1H-rDdc7VOtvEVpUJ82TwyQ/rs:fill:1000:1000:1:0/plain/bafkreiaivizp4xldojmmpuzmiu75cmea7nq56dnntnuhzhsjcb63aou5ei@jpeg", + "did": "user(3)", + "displayName": "bobby", + "handle": "bob.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "followedBy": "record(8)", + "following": "record(7)", + "muted": false, + }, + }, + "cid": "cids(10)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 0, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000+00:00", + "text": "bobby boy here", + }, + "replyCount": 0, + "repostCount": 0, + "uri": "record(13)", + "viewer": Object {}, + }, + }, + Object { + "post": Object { + "author": Object { + "avatar": "https://pds.public.url/image/KzkHFsMRQ6oAKCHCRKFA1H-rDdc7VOtvEVpUJ82TwyQ/rs:fill:1000:1000:1:0/plain/bafkreiaivizp4xldojmmpuzmiu75cmea7nq56dnntnuhzhsjcb63aou5ei@jpeg", + "did": "user(0)", + "displayName": "ali", + "handle": "alice.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "muted": false, + }, + }, + "cid": "cids(0)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 3, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000000Z", + "text": "again", + }, + "replyCount": 2, + "repostCount": 1, + "uri": "record(0)", + "viewer": Object {}, + }, + }, + Object { + "post": Object { + "author": Object { + "did": "user(1)", + "handle": "dan.test", + "labels": Array [ + Object { + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "did:example:labeler", + "uri": "user(1)", + "val": "repo-action-label", + }, + ], + "viewer": Object { + "blockedBy": false, + "following": "record(1)", + "muted": false, + }, + }, + "cid": "cids(1)", + "embed": Object { + "$type": "app.bsky.embed.record#view", + "record": Object { + "$type": "app.bsky.embed.record#viewRecord", + "author": Object { + "did": "user(2)", + "handle": "carol.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "followedBy": "record(5)", + "following": "record(4)", + "muted": false, + }, + }, + "cid": "cids(2)", + "embeds": Array [ + Object { + "$type": "app.bsky.embed.recordWithMedia#view", + "media": Object { + "$type": "app.bsky.embed.images#view", + "images": Array [ + Object { + "alt": "tests/image/fixtures/key-landscape-small.jpg", + "fullsize": "https://pds.public.url/image/AiDXkxVbgBksxb1nfiRn1m6S4K8_mee6o8r-UGLNzOM/rs:fit:2000:2000:1:0/plain/bafkreigy5p3xxceipk2o6nqtnugpft26ol6yleqhboqziino7axvdngtci@jpeg", + "thumb": "https://pds.public.url/image/uc7FGfiGv0mMqmk9XiqHXrIhNymLHaex7Ge8nEhmXqo/rs:fit:1000:1000:1:0/plain/bafkreigy5p3xxceipk2o6nqtnugpft26ol6yleqhboqziino7axvdngtci@jpeg", + }, + Object { + "alt": "tests/image/fixtures/key-alt.jpg", + "fullsize": "https://pds.public.url/image/xC2No-8rKVDIwIMmCiEBm9EiGLDBBOpf36PHoGf-GDw/rs:fit:2000:2000:1:0/plain/bafkreifdklbbcdsyanjz3oqe5pf2omuq5ansthokxlbleagg3eenx62h7e@jpeg", + "thumb": "https://pds.public.url/image/g7yazUpNwN8LKumZ2Zmn_ptQbtMLs1Pti5-GDn7H8_8/rs:fit:1000:1000:1:0/plain/bafkreifdklbbcdsyanjz3oqe5pf2omuq5ansthokxlbleagg3eenx62h7e@jpeg", + }, + ], + }, + "record": Object { + "record": Object { + "$type": "app.bsky.embed.record#viewRecord", + "author": Object { + "avatar": "https://pds.public.url/image/KzkHFsMRQ6oAKCHCRKFA1H-rDdc7VOtvEVpUJ82TwyQ/rs:fill:1000:1000:1:0/plain/bafkreiaivizp4xldojmmpuzmiu75cmea7nq56dnntnuhzhsjcb63aou5ei@jpeg", + "did": "user(3)", + "displayName": "bobby", + "handle": "bob.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "followedBy": "record(8)", + "following": "record(7)", + "muted": false, + }, + }, + "cid": "cids(5)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "uri": "record(6)", + "value": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "langs": Array [ + "en-US", + "i-klingon", + ], + "text": "bob back at it again!", + }, + }, + }, + }, + ], + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "uri": "record(3)", + "value": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "embed": Object { + "$type": "app.bsky.embed.recordWithMedia", + "media": Object { + "$type": "app.bsky.embed.images", + "images": Array [ + Object { + "alt": "tests/image/fixtures/key-landscape-small.jpg", + "image": Object { + "$type": "blob", + "mimeType": "image/jpeg", + "ref": Object { + "$link": "cids(3)", + }, + "size": 4114, + }, + }, + Object { + "alt": "tests/image/fixtures/key-alt.jpg", + "image": Object { + "$type": "blob", + "mimeType": "image/jpeg", + "ref": Object { + "$link": "cids(4)", + }, + "size": 12736, + }, + }, + ], + }, + "record": Object { + "record": Object { + "cid": "cids(5)", + "uri": "record(6)", + }, + }, + }, + "text": "hi im carol", + }, + }, + }, + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 0, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "embed": Object { + "$type": "app.bsky.embed.record", + "record": Object { + "cid": "cids(2)", + "uri": "record(3)", + }, + }, + "facets": Array [ + Object { + "features": Array [ + Object { + "$type": "app.bsky.richtext.facet#mention", + "did": "user(0)", + }, + ], + "index": Object { + "byteEnd": 18, + "byteStart": 0, + }, + }, + ], + "text": "@alice.bluesky.xyz is the best", + }, + "replyCount": 0, + "repostCount": 1, + "uri": "record(2)", + "viewer": Object {}, + }, + }, + Object { + "post": Object { + "author": Object { + "did": "user(1)", + "handle": "dan.test", + "labels": Array [ + Object { + "cts": "1970-01-01T00:00:00.000Z", + "neg": false, + "src": "did:example:labeler", + "uri": "user(1)", + "val": "repo-action-label", + }, + ], + "viewer": Object { + "blockedBy": false, + "following": "record(1)", + "muted": false, + }, + }, + "cid": "cids(11)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 0, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "text": "dan here!", + }, + "replyCount": 0, + "repostCount": 0, + "uri": "record(14)", + "viewer": Object {}, + }, + }, + Object { + "post": Object { + "author": Object { + "did": "user(2)", + "handle": "carol.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "followedBy": "record(5)", + "following": "record(4)", + "muted": false, + }, + }, + "cid": "cids(2)", + "embed": Object { + "$type": "app.bsky.embed.recordWithMedia#view", + "media": Object { + "$type": "app.bsky.embed.images#view", + "images": Array [ + Object { + "alt": "tests/image/fixtures/key-landscape-small.jpg", + "fullsize": "https://pds.public.url/image/AiDXkxVbgBksxb1nfiRn1m6S4K8_mee6o8r-UGLNzOM/rs:fit:2000:2000:1:0/plain/bafkreigy5p3xxceipk2o6nqtnugpft26ol6yleqhboqziino7axvdngtci@jpeg", + "thumb": "https://pds.public.url/image/uc7FGfiGv0mMqmk9XiqHXrIhNymLHaex7Ge8nEhmXqo/rs:fit:1000:1000:1:0/plain/bafkreigy5p3xxceipk2o6nqtnugpft26ol6yleqhboqziino7axvdngtci@jpeg", + }, + Object { + "alt": "tests/image/fixtures/key-alt.jpg", + "fullsize": "https://pds.public.url/image/xC2No-8rKVDIwIMmCiEBm9EiGLDBBOpf36PHoGf-GDw/rs:fit:2000:2000:1:0/plain/bafkreifdklbbcdsyanjz3oqe5pf2omuq5ansthokxlbleagg3eenx62h7e@jpeg", + "thumb": "https://pds.public.url/image/g7yazUpNwN8LKumZ2Zmn_ptQbtMLs1Pti5-GDn7H8_8/rs:fit:1000:1000:1:0/plain/bafkreifdklbbcdsyanjz3oqe5pf2omuq5ansthokxlbleagg3eenx62h7e@jpeg", + }, + ], + }, + "record": Object { + "record": Object { + "$type": "app.bsky.embed.record#viewRecord", + "author": Object { + "avatar": "https://pds.public.url/image/KzkHFsMRQ6oAKCHCRKFA1H-rDdc7VOtvEVpUJ82TwyQ/rs:fill:1000:1000:1:0/plain/bafkreiaivizp4xldojmmpuzmiu75cmea7nq56dnntnuhzhsjcb63aou5ei@jpeg", + "did": "user(3)", + "displayName": "bobby", + "handle": "bob.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "followedBy": "record(8)", + "following": "record(7)", + "muted": false, + }, + }, + "cid": "cids(5)", + "embeds": Array [], + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "uri": "record(6)", + "value": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "langs": Array [ + "en-US", + "i-klingon", + ], + "text": "bob back at it again!", + }, + }, + }, + }, + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 2, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "embed": Object { + "$type": "app.bsky.embed.recordWithMedia", + "media": Object { + "$type": "app.bsky.embed.images", + "images": Array [ + Object { + "alt": "tests/image/fixtures/key-landscape-small.jpg", + "image": Object { + "$type": "blob", + "mimeType": "image/jpeg", + "ref": Object { + "$link": "cids(3)", + }, + "size": 4114, + }, + }, + Object { + "alt": "tests/image/fixtures/key-alt.jpg", + "image": Object { + "$type": "blob", + "mimeType": "image/jpeg", + "ref": Object { + "$link": "cids(4)", + }, + "size": 12736, + }, + }, + ], + }, + "record": Object { + "record": Object { + "cid": "cids(5)", + "uri": "record(6)", + }, + }, + }, + "text": "hi im carol", + }, + "replyCount": 0, + "repostCount": 0, + "uri": "record(3)", + "viewer": Object { + "like": "record(15)", + }, + }, + }, + Object { + "post": Object { + "author": Object { + "avatar": "https://pds.public.url/image/KzkHFsMRQ6oAKCHCRKFA1H-rDdc7VOtvEVpUJ82TwyQ/rs:fill:1000:1000:1:0/plain/bafkreiaivizp4xldojmmpuzmiu75cmea7nq56dnntnuhzhsjcb63aou5ei@jpeg", + "did": "user(3)", + "displayName": "bobby", + "handle": "bob.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "followedBy": "record(8)", + "following": "record(7)", + "muted": false, + }, + }, + "cid": "cids(5)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 0, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "langs": Array [ + "en-US", + "i-klingon", + ], + "text": "bob back at it again!", + }, + "replyCount": 0, + "repostCount": 0, + "uri": "record(6)", + "viewer": Object {}, + }, + }, + Object { + "post": Object { + "author": Object { + "avatar": "https://pds.public.url/image/KzkHFsMRQ6oAKCHCRKFA1H-rDdc7VOtvEVpUJ82TwyQ/rs:fill:1000:1000:1:0/plain/bafkreiaivizp4xldojmmpuzmiu75cmea7nq56dnntnuhzhsjcb63aou5ei@jpeg", + "did": "user(0)", + "displayName": "ali", + "handle": "alice.test", + "labels": Array [], + "viewer": Object { + "blockedBy": false, + "muted": false, + }, + }, + "cid": "cids(12)", + "indexedAt": "1970-01-01T00:00:00.000Z", + "labels": Array [], + "likeCount": 0, + "record": Object { + "$type": "app.bsky.feed.post", + "createdAt": "1970-01-01T00:00:00.000Z", + "text": "hey there", + }, + "replyCount": 0, + "repostCount": 0, + "uri": "record(16)", + "viewer": Object {}, + }, + }, + ], +} +`; diff --git a/packages/pds/tests/proxied/timeline-skeleton.test.ts b/packages/pds/tests/proxied/timeline-skeleton.test.ts new file mode 100644 index 00000000000..680dd3423d4 --- /dev/null +++ b/packages/pds/tests/proxied/timeline-skeleton.test.ts @@ -0,0 +1,59 @@ +import AtpAgent from '@atproto/api' +import { TestNetwork } from '@atproto/dev-env' +import { SeedClient } from '../seeds/client' +import basicSeed from '../seeds/basic' +import { forSnapshot } from '../_util' + +describe('proxies timeline skeleton', () => { + let network: TestNetwork + let agent: AtpAgent + let sc: SeedClient + + let alice: string + + beforeAll(async () => { + network = await TestNetwork.create({ + dbPostgresSchema: 'proxy_timeline_skeleton', + pds: { + enableAppView: true, + }, + }) + agent = network.pds.getClient() + sc = new SeedClient(agent) + await basicSeed(sc) + await network.processAll() + alice = sc.dids.alice + }) + + afterAll(async () => { + await network.close() + }) + + it('timeline skeleton construction', async () => { + const res = await agent.api.app.bsky.feed.getTimeline( + {}, + { + headers: { ...sc.getHeaders(alice), 'x-appview-proxy': 'true' }, + }, + ) + + expect(forSnapshot(res.data)).toMatchSnapshot() + const pt1 = await agent.api.app.bsky.feed.getTimeline( + { + limit: 2, + }, + { + headers: { ...sc.getHeaders(alice), 'x-appview-proxy': 'true' }, + }, + ) + const pt2 = await agent.api.app.bsky.feed.getTimeline( + { + cursor: pt1.data.cursor, + }, + { + headers: { ...sc.getHeaders(alice), 'x-appview-proxy': 'true' }, + }, + ) + expect([...pt1.data.feed, ...pt2.data.feed]).toEqual(res.data.feed) + }) +}) diff --git a/packages/pds/tests/proxied/views.test.ts b/packages/pds/tests/proxied/views.test.ts index ff5336c3e5f..33166b9de78 100644 --- a/packages/pds/tests/proxied/views.test.ts +++ b/packages/pds/tests/proxied/views.test.ts @@ -257,13 +257,15 @@ describe('proxies view requests', () => { expect(forSnapshot(res.data)).toMatchSnapshot() }) - it('feed.getTimeline', async () => { + // @TODO re-enable when proxying is a full-proxy + it.skip('feed.getTimeline', async () => { const res = await agent.api.app.bsky.feed.getTimeline( {}, { headers: { ...sc.getHeaders(alice), 'x-appview-proxy': 'true' }, }, ) + expect(forSnapshot(res.data)).toMatchSnapshot() const pt1 = await agent.api.app.bsky.feed.getTimeline( { From 6695e6c9e404685c3711ce4bf27fa412f5a612d3 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Mon, 3 Jul 2023 12:38:33 -0500 Subject: [PATCH 03/13] Only pass through known params on timeline skeleton (#1270) only pass through own params --- packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts b/packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts index c6b0bd9e7a4..1b3e6271b50 100644 --- a/packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts +++ b/packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts @@ -19,7 +19,7 @@ export default function (server: Server, ctx: AppContext) { if (ctx.canProxy(req)) { const res = await ctx.appviewAgent.api.app.bsky.unspecced.getTimelineSkeleton( - params, + { limit, cursor }, await ctx.serviceAuthHeaders(requester), ) const filtered = await filterMutesAndBlocks( From 9610ba061c39bdc6048bbf929fe4842a870b72e8 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Mon, 3 Jul 2023 13:19:56 -0500 Subject: [PATCH 04/13] Require headers on getRecord proxy (#1271) require headers on getRecord proxy --- packages/pds/src/api/com/atproto/repo/getRecord.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pds/src/api/com/atproto/repo/getRecord.ts b/packages/pds/src/api/com/atproto/repo/getRecord.ts index 163411360f6..020935eaffa 100644 --- a/packages/pds/src/api/com/atproto/repo/getRecord.ts +++ b/packages/pds/src/api/com/atproto/repo/getRecord.ts @@ -4,7 +4,7 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' export default function (server: Server, ctx: AppContext) { - server.com.atproto.repo.getRecord(async ({ params }) => { + server.com.atproto.repo.getRecord(async ({ req, params }) => { const { repo, collection, rkey, cid } = params const did = await ctx.services.account(ctx.db).getDidForActor(repo) @@ -26,7 +26,7 @@ export default function (server: Server, ctx: AppContext) { } } - if (ctx.cfg.bskyAppViewEndpoint) { + if (ctx.canProxy(req)) { const res = await ctx.appviewAgent.api.com.atproto.repo.getRecord(params) return { encoding: 'application/json', From c793ff9103c1348fbc80c223d16e1ae8bc997b0a Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Mon, 3 Jul 2023 14:14:49 -0500 Subject: [PATCH 05/13] Add boolean for enabling generic appview proxying (#1273) * add boolean config for enabling generic proxying * tweak * tweak cfg var name * tweak --- packages/dev-env/src/pds.ts | 7 ++++++- packages/dev-env/src/types.ts | 2 +- .../api/com/atproto/identity/resolveHandle.ts | 4 ++-- .../pds/src/api/com/atproto/repo/getRecord.ts | 2 +- .../app-view/api/app/bsky/actor/getProfile.ts | 2 +- .../app-view/api/app/bsky/actor/getProfiles.ts | 2 +- .../api/app/bsky/actor/getSuggestions.ts | 2 +- .../app-view/api/app/bsky/actor/searchActors.ts | 2 +- .../api/app/bsky/actor/searchActorsTypeahead.ts | 2 +- .../app-view/api/app/bsky/feed/getActorFeeds.ts | 2 +- .../app-view/api/app/bsky/feed/getAuthorFeed.ts | 2 +- .../src/app-view/api/app/bsky/feed/getFeed.ts | 2 +- .../api/app/bsky/feed/getFeedGenerator.ts | 2 +- .../api/app/bsky/feed/getFeedGenerators.ts | 2 +- .../src/app-view/api/app/bsky/feed/getLikes.ts | 2 +- .../app-view/api/app/bsky/feed/getPostThread.ts | 2 +- .../src/app-view/api/app/bsky/feed/getPosts.ts | 2 +- .../app-view/api/app/bsky/feed/getRepostedBy.ts | 2 +- .../app-view/api/app/bsky/feed/getTimeline.ts | 12 +++++++++++- .../src/app-view/api/app/bsky/graph/getBlocks.ts | 2 +- .../app-view/api/app/bsky/graph/getFollowers.ts | 2 +- .../app-view/api/app/bsky/graph/getFollows.ts | 2 +- .../src/app-view/api/app/bsky/graph/getList.ts | 2 +- .../app-view/api/app/bsky/graph/getListMutes.ts | 2 +- .../src/app-view/api/app/bsky/graph/getLists.ts | 2 +- .../src/app-view/api/app/bsky/graph/getMutes.ts | 2 +- .../src/app-view/api/app/bsky/graph/muteActor.ts | 2 +- .../app-view/api/app/bsky/graph/muteActorList.ts | 2 +- .../app-view/api/app/bsky/graph/unmuteActor.ts | 2 +- .../api/app/bsky/graph/unmuteActorList.ts | 2 +- .../api/app/bsky/notification/getUnreadCount.ts | 2 +- .../app/bsky/notification/listNotifications.ts | 2 +- .../api/app/bsky/notification/updateSeen.ts | 2 +- .../pds/src/app-view/api/app/bsky/unspecced.ts | 2 +- packages/pds/src/config.ts | 8 ++++++++ packages/pds/src/context.ts | 16 +++++++++++++++- packages/pds/tests/_util.ts | 1 + .../pds/tests/proxied/timeline-skeleton.test.ts | 3 ++- 38 files changed, 76 insertions(+), 37 deletions(-) diff --git a/packages/dev-env/src/pds.ts b/packages/dev-env/src/pds.ts index 6a818e4e235..ee0c307f141 100644 --- a/packages/dev-env/src/pds.ts +++ b/packages/dev-env/src/pds.ts @@ -64,6 +64,7 @@ export class TestPds { labelerKeywords: { label_me: 'test-label', label_me_2: 'test-label-2' }, feedGenDid: 'did:example:feedGen', dbTxLockNonce: await randomStr(32, 'base32'), + bskyAppViewProxy: !!cfg.bskyAppViewEndpoint, ...cfg, }) @@ -77,7 +78,11 @@ export class TestPds { : pds.Database.memory() await db.migrateToLatestOrThrow() - if (config.bskyAppViewEndpoint && !cfg.enableAppView) { + if ( + config.bskyAppViewEndpoint && + config.bskyAppViewProxy && + !cfg.enableInProcessAppView + ) { // Disable communication to app view within pds MessageDispatcher.prototype.send = async () => {} } diff --git a/packages/dev-env/src/types.ts b/packages/dev-env/src/types.ts index 408ce8cee4f..829d7f7518e 100644 --- a/packages/dev-env/src/types.ts +++ b/packages/dev-env/src/types.ts @@ -9,7 +9,7 @@ export type PlcConfig = { export type PdsConfig = Partial & { plcUrl: string migration?: string - enableAppView?: boolean + enableInProcessAppView?: boolean } export type BskyConfig = Partial & { diff --git a/packages/pds/src/api/com/atproto/identity/resolveHandle.ts b/packages/pds/src/api/com/atproto/identity/resolveHandle.ts index f2c222e665d..bfa47b47764 100644 --- a/packages/pds/src/api/com/atproto/identity/resolveHandle.ts +++ b/packages/pds/src/api/com/atproto/identity/resolveHandle.ts @@ -5,7 +5,7 @@ import { Server } from '../../../../lexicon' import AppContext from '../../../../context' export default function (server: Server, ctx: AppContext) { - server.com.atproto.identity.resolveHandle(async ({ params }) => { + server.com.atproto.identity.resolveHandle(async ({ params, req }) => { let handle: string try { handle = ident.normalizeAndEnsureValidHandle(params.handle) @@ -34,7 +34,7 @@ export default function (server: Server, ctx: AppContext) { // this is not someone on our server, but we help with resolving anyway - if (!did && ctx.cfg.bskyAppViewEndpoint) { + if (!did && ctx.canProxyRead(req)) { did = await tryResolveFromAppview(ctx.appviewAgent, handle) } diff --git a/packages/pds/src/api/com/atproto/repo/getRecord.ts b/packages/pds/src/api/com/atproto/repo/getRecord.ts index 020935eaffa..bec4fff1e8c 100644 --- a/packages/pds/src/api/com/atproto/repo/getRecord.ts +++ b/packages/pds/src/api/com/atproto/repo/getRecord.ts @@ -26,7 +26,7 @@ export default function (server: Server, ctx: AppContext) { } } - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.com.atproto.repo.getRecord(params) return { encoding: 'application/json', diff --git a/packages/pds/src/app-view/api/app/bsky/actor/getProfile.ts b/packages/pds/src/app-view/api/app/bsky/actor/getProfile.ts index 850bd1015de..df37174f7e5 100644 --- a/packages/pds/src/app-view/api/app/bsky/actor/getProfile.ts +++ b/packages/pds/src/app-view/api/app/bsky/actor/getProfile.ts @@ -8,7 +8,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, auth, params }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.actor.getProfile( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/actor/getProfiles.ts b/packages/pds/src/app-view/api/app/bsky/actor/getProfiles.ts index bcc129b8de2..673868df384 100644 --- a/packages/pds/src/app-view/api/app/bsky/actor/getProfiles.ts +++ b/packages/pds/src/app-view/api/app/bsky/actor/getProfiles.ts @@ -6,7 +6,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, auth, params }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.actor.getProfiles( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/actor/getSuggestions.ts b/packages/pds/src/app-view/api/app/bsky/actor/getSuggestions.ts index 7b7f8d1289d..540f7795122 100644 --- a/packages/pds/src/app-view/api/app/bsky/actor/getSuggestions.ts +++ b/packages/pds/src/app-view/api/app/bsky/actor/getSuggestions.ts @@ -7,7 +7,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.actor.getSuggestions( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/actor/searchActors.ts b/packages/pds/src/app-view/api/app/bsky/actor/searchActors.ts index f3f95460686..380fea56ae4 100644 --- a/packages/pds/src/app-view/api/app/bsky/actor/searchActors.ts +++ b/packages/pds/src/app-view/api/app/bsky/actor/searchActors.ts @@ -16,7 +16,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, auth, params }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.actor.searchActors( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/actor/searchActorsTypeahead.ts b/packages/pds/src/app-view/api/app/bsky/actor/searchActorsTypeahead.ts index f0868a402ec..2cf879af954 100644 --- a/packages/pds/src/app-view/api/app/bsky/actor/searchActorsTypeahead.ts +++ b/packages/pds/src/app-view/api/app/bsky/actor/searchActorsTypeahead.ts @@ -14,7 +14,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.actor.searchActorsTypeahead( params, diff --git a/packages/pds/src/app-view/api/app/bsky/feed/getActorFeeds.ts b/packages/pds/src/app-view/api/app/bsky/feed/getActorFeeds.ts index fe6ac23416f..6cc8becbeac 100644 --- a/packages/pds/src/app-view/api/app/bsky/feed/getActorFeeds.ts +++ b/packages/pds/src/app-view/api/app/bsky/feed/getActorFeeds.ts @@ -8,7 +8,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, auth, params }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.feed.getActorFeeds( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/feed/getAuthorFeed.ts b/packages/pds/src/app-view/api/app/bsky/feed/getAuthorFeed.ts index f30e5d2773a..6a349dd6ccf 100644 --- a/packages/pds/src/app-view/api/app/bsky/feed/getAuthorFeed.ts +++ b/packages/pds/src/app-view/api/app/bsky/feed/getAuthorFeed.ts @@ -10,7 +10,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.feed.getAuthorFeed( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/feed/getFeed.ts b/packages/pds/src/app-view/api/app/bsky/feed/getFeed.ts index e521f4fda34..ca5c09ce39e 100644 --- a/packages/pds/src/app-view/api/app/bsky/feed/getFeed.ts +++ b/packages/pds/src/app-view/api/app/bsky/feed/getFeed.ts @@ -25,7 +25,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const { data: feed } = await ctx.appviewAgent.api.app.bsky.feed.getFeedGenerator( { feed: params.feed }, diff --git a/packages/pds/src/app-view/api/app/bsky/feed/getFeedGenerator.ts b/packages/pds/src/app-view/api/app/bsky/feed/getFeedGenerator.ts index ec874565ebe..7f4de6fa327 100644 --- a/packages/pds/src/app-view/api/app/bsky/feed/getFeedGenerator.ts +++ b/packages/pds/src/app-view/api/app/bsky/feed/getFeedGenerator.ts @@ -12,7 +12,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.feed.getFeedGenerator( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/feed/getFeedGenerators.ts b/packages/pds/src/app-view/api/app/bsky/feed/getFeedGenerators.ts index a7879dd0934..0e504abd39d 100644 --- a/packages/pds/src/app-view/api/app/bsky/feed/getFeedGenerators.ts +++ b/packages/pds/src/app-view/api/app/bsky/feed/getFeedGenerators.ts @@ -6,7 +6,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.feed.getFeedGenerators( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/feed/getLikes.ts b/packages/pds/src/app-view/api/app/bsky/feed/getLikes.ts index a29f69a8be7..262dd678f39 100644 --- a/packages/pds/src/app-view/api/app/bsky/feed/getLikes.ts +++ b/packages/pds/src/app-view/api/app/bsky/feed/getLikes.ts @@ -8,7 +8,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.feed.getLikes( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/feed/getPostThread.ts b/packages/pds/src/app-view/api/app/bsky/feed/getPostThread.ts index 90244bf2e3f..da8f72dfde2 100644 --- a/packages/pds/src/app-view/api/app/bsky/feed/getPostThread.ts +++ b/packages/pds/src/app-view/api/app/bsky/feed/getPostThread.ts @@ -27,7 +27,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.feed.getPostThread( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/feed/getPosts.ts b/packages/pds/src/app-view/api/app/bsky/feed/getPosts.ts index cc6c39ec44b..2572702d78d 100644 --- a/packages/pds/src/app-view/api/app/bsky/feed/getPosts.ts +++ b/packages/pds/src/app-view/api/app/bsky/feed/getPosts.ts @@ -8,7 +8,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.feed.getPosts( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/feed/getRepostedBy.ts b/packages/pds/src/app-view/api/app/bsky/feed/getRepostedBy.ts index 5d5948954b5..4632883d0ef 100644 --- a/packages/pds/src/app-view/api/app/bsky/feed/getRepostedBy.ts +++ b/packages/pds/src/app-view/api/app/bsky/feed/getRepostedBy.ts @@ -8,7 +8,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.feed.getRepostedBy( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts b/packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts index 1b3e6271b50..3587c3acc7d 100644 --- a/packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts +++ b/packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts @@ -16,7 +16,17 @@ export default function (server: Server, ctx: AppContext) { throw new InvalidRequestError(`Unsupported algorithm: ${algorithm}`) } - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { + const res = await ctx.appviewAgent.api.app.bsky.feed.getTimeline( + params, + await ctx.serviceAuthHeaders(requester), + ) + return { + encoding: 'application/json', + body: res.data, + } + } + if (ctx.canProxyFeedConstruction(req)) { const res = await ctx.appviewAgent.api.app.bsky.unspecced.getTimelineSkeleton( { limit, cursor }, diff --git a/packages/pds/src/app-view/api/app/bsky/graph/getBlocks.ts b/packages/pds/src/app-view/api/app/bsky/graph/getBlocks.ts index e14a12bd37a..e76eb2963a1 100644 --- a/packages/pds/src/app-view/api/app/bsky/graph/getBlocks.ts +++ b/packages/pds/src/app-view/api/app/bsky/graph/getBlocks.ts @@ -8,7 +8,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.graph.getBlocks( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/graph/getFollowers.ts b/packages/pds/src/app-view/api/app/bsky/graph/getFollowers.ts index 3b3281edb41..93bf73075e0 100644 --- a/packages/pds/src/app-view/api/app/bsky/graph/getFollowers.ts +++ b/packages/pds/src/app-view/api/app/bsky/graph/getFollowers.ts @@ -9,7 +9,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.graph.getFollowers( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/graph/getFollows.ts b/packages/pds/src/app-view/api/app/bsky/graph/getFollows.ts index 4a8bf888fc6..75edf58c225 100644 --- a/packages/pds/src/app-view/api/app/bsky/graph/getFollows.ts +++ b/packages/pds/src/app-view/api/app/bsky/graph/getFollows.ts @@ -9,7 +9,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.graph.getFollows( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/graph/getList.ts b/packages/pds/src/app-view/api/app/bsky/graph/getList.ts index 77b16e9677c..c711ea93026 100644 --- a/packages/pds/src/app-view/api/app/bsky/graph/getList.ts +++ b/packages/pds/src/app-view/api/app/bsky/graph/getList.ts @@ -9,7 +9,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.graph.getList( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/graph/getListMutes.ts b/packages/pds/src/app-view/api/app/bsky/graph/getListMutes.ts index e3a486ccf0b..89501548f40 100644 --- a/packages/pds/src/app-view/api/app/bsky/graph/getListMutes.ts +++ b/packages/pds/src/app-view/api/app/bsky/graph/getListMutes.ts @@ -8,7 +8,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.graph.getListMutes( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/graph/getLists.ts b/packages/pds/src/app-view/api/app/bsky/graph/getLists.ts index cb77e2bf2e8..7dc77e6c5ef 100644 --- a/packages/pds/src/app-view/api/app/bsky/graph/getLists.ts +++ b/packages/pds/src/app-view/api/app/bsky/graph/getLists.ts @@ -8,7 +8,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.graph.getLists( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/graph/getMutes.ts b/packages/pds/src/app-view/api/app/bsky/graph/getMutes.ts index 98218309800..51950515cd4 100644 --- a/packages/pds/src/app-view/api/app/bsky/graph/getMutes.ts +++ b/packages/pds/src/app-view/api/app/bsky/graph/getMutes.ts @@ -8,7 +8,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, auth, params }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.graph.getMutes( params, await ctx.serviceAuthHeaders(requester), diff --git a/packages/pds/src/app-view/api/app/bsky/graph/muteActor.ts b/packages/pds/src/app-view/api/app/bsky/graph/muteActor.ts index ce3dfcf4125..ac22ebd2bba 100644 --- a/packages/pds/src/app-view/api/app/bsky/graph/muteActor.ts +++ b/packages/pds/src/app-view/api/app/bsky/graph/muteActor.ts @@ -23,7 +23,7 @@ export default function (server: Server, ctx: AppContext) { mutedByDid: requester, }) - if (ctx.cfg.bskyAppViewEndpoint) { + if (ctx.canProxyWrite()) { await ctx.appviewAgent.api.app.bsky.graph.muteActor(input.body, { ...(await ctx.serviceAuthHeaders(requester)), encoding: 'application/json', diff --git a/packages/pds/src/app-view/api/app/bsky/graph/muteActorList.ts b/packages/pds/src/app-view/api/app/bsky/graph/muteActorList.ts index 982a652f7b8..65c09125f45 100644 --- a/packages/pds/src/app-view/api/app/bsky/graph/muteActorList.ts +++ b/packages/pds/src/app-view/api/app/bsky/graph/muteActorList.ts @@ -22,7 +22,7 @@ export default function (server: Server, ctx: AppContext) { mutedByDid: requester, }) - if (ctx.cfg.bskyAppViewEndpoint) { + if (ctx.canProxyWrite()) { await ctx.appviewAgent.api.app.bsky.graph.muteActorList(input.body, { ...(await ctx.serviceAuthHeaders(requester)), encoding: 'application/json', diff --git a/packages/pds/src/app-view/api/app/bsky/graph/unmuteActor.ts b/packages/pds/src/app-view/api/app/bsky/graph/unmuteActor.ts index ae9e8dac742..1b911e71336 100644 --- a/packages/pds/src/app-view/api/app/bsky/graph/unmuteActor.ts +++ b/packages/pds/src/app-view/api/app/bsky/graph/unmuteActor.ts @@ -20,7 +20,7 @@ export default function (server: Server, ctx: AppContext) { mutedByDid: requester, }) - if (ctx.cfg.bskyAppViewEndpoint) { + if (ctx.canProxyWrite()) { await ctx.appviewAgent.api.app.bsky.graph.unmuteActor(input.body, { ...(await ctx.serviceAuthHeaders(requester)), encoding: 'application/json', diff --git a/packages/pds/src/app-view/api/app/bsky/graph/unmuteActorList.ts b/packages/pds/src/app-view/api/app/bsky/graph/unmuteActorList.ts index 69012072100..36e65e4178a 100644 --- a/packages/pds/src/app-view/api/app/bsky/graph/unmuteActorList.ts +++ b/packages/pds/src/app-view/api/app/bsky/graph/unmuteActorList.ts @@ -13,7 +13,7 @@ export default function (server: Server, ctx: AppContext) { mutedByDid: requester, }) - if (ctx.cfg.bskyAppViewEndpoint) { + if (ctx.canProxyWrite()) { await ctx.appviewAgent.api.app.bsky.graph.unmuteActorList(input.body, { ...(await ctx.serviceAuthHeaders(requester)), encoding: 'application/json', diff --git a/packages/pds/src/app-view/api/app/bsky/notification/getUnreadCount.ts b/packages/pds/src/app-view/api/app/bsky/notification/getUnreadCount.ts index e16e2be46ce..c1b7c276d9f 100644 --- a/packages/pds/src/app-view/api/app/bsky/notification/getUnreadCount.ts +++ b/packages/pds/src/app-view/api/app/bsky/notification/getUnreadCount.ts @@ -8,7 +8,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, auth, params }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.notification.getUnreadCount( params, diff --git a/packages/pds/src/app-view/api/app/bsky/notification/listNotifications.ts b/packages/pds/src/app-view/api/app/bsky/notification/listNotifications.ts index c9ec4885e74..2967c069ad9 100644 --- a/packages/pds/src/app-view/api/app/bsky/notification/listNotifications.ts +++ b/packages/pds/src/app-view/api/app/bsky/notification/listNotifications.ts @@ -11,7 +11,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const res = await ctx.appviewAgent.api.app.bsky.notification.listNotifications( params, diff --git a/packages/pds/src/app-view/api/app/bsky/notification/updateSeen.ts b/packages/pds/src/app-view/api/app/bsky/notification/updateSeen.ts index 717f358f77c..f5644abfe7f 100644 --- a/packages/pds/src/app-view/api/app/bsky/notification/updateSeen.ts +++ b/packages/pds/src/app-view/api/app/bsky/notification/updateSeen.ts @@ -27,7 +27,7 @@ export default function (server: Server, ctx: AppContext) { .where('did', '=', user.did) .executeTakeFirst() - if (ctx.cfg.bskyAppViewEndpoint) { + if (ctx.canProxyWrite()) { await ctx.appviewAgent.api.app.bsky.notification.updateSeen( input.body, { diff --git a/packages/pds/src/app-view/api/app/bsky/unspecced.ts b/packages/pds/src/app-view/api/app/bsky/unspecced.ts index 12b6db691eb..5688549fbe7 100644 --- a/packages/pds/src/app-view/api/app/bsky/unspecced.ts +++ b/packages/pds/src/app-view/api/app/bsky/unspecced.ts @@ -23,7 +23,7 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifier, handler: async ({ req, params, auth }) => { const requester = auth.credentials.did - if (ctx.canProxy(req)) { + if (ctx.canProxyRead(req)) { const hotClassicUri = Object.keys(ctx.algos).find((uri) => uri.endsWith('/hot-classic'), ) diff --git a/packages/pds/src/config.ts b/packages/pds/src/config.ts index c884721b1e5..b860cb75908 100644 --- a/packages/pds/src/config.ts +++ b/packages/pds/src/config.ts @@ -60,6 +60,7 @@ export interface ServerConfigValues { bskyAppViewEndpoint?: string bskyAppViewDid?: string + bskyAppViewProxy: boolean crawlersToNotify?: string[] } @@ -181,6 +182,8 @@ export class ServerConfig { process.env.BSKY_APP_VIEW_ENDPOINT, ) const bskyAppViewDid = nonemptyString(process.env.BSKY_APP_VIEW_DID) + const bskyAppViewProxy = + process.env.BSKY_APP_VIEW_PROXY === 'true' ? true : false const crawlersEnv = process.env.CRAWLERS_TO_NOTIFY const crawlersToNotify = @@ -229,6 +232,7 @@ export class ServerConfig { dbTxLockNonce, bskyAppViewEndpoint, bskyAppViewDid, + bskyAppViewProxy, crawlersToNotify, ...overrides, }) @@ -432,6 +436,10 @@ export class ServerConfig { return this.cfg.bskyAppViewDid } + get bskyAppViewProxy() { + return this.cfg.bskyAppViewProxy + } + get crawlersToNotify() { return this.cfg.crawlersToNotify } diff --git a/packages/pds/src/context.ts b/packages/pds/src/context.ts index 75128a0d0ae..1e3ce8167d4 100644 --- a/packages/pds/src/context.ts +++ b/packages/pds/src/context.ts @@ -174,12 +174,26 @@ export class AppContext { return this._appviewAgent } - canProxy(req: express.Request): boolean { + canProxyRead(req: express.Request): boolean { return ( + this.cfg.bskyAppViewProxy && this.cfg.bskyAppViewEndpoint !== undefined && req.get('x-appview-proxy') !== undefined ) } + + canProxyFeedConstruction(req: express.Request): boolean { + return ( + this.cfg.bskyAppViewEndpoint !== undefined && + req.get('x-appview-proxy') !== undefined + ) + } + + canProxyWrite(): boolean { + return ( + this.cfg.bskyAppViewProxy && this.cfg.bskyAppViewEndpoint !== undefined + ) + } } export default AppContext diff --git a/packages/pds/tests/_util.ts b/packages/pds/tests/_util.ts index b12c3cffad3..225b3a85a1d 100644 --- a/packages/pds/tests/_util.ts +++ b/packages/pds/tests/_util.ts @@ -106,6 +106,7 @@ export const runTestServer = async ( repoBackfillLimitMs: HOUR, sequencerLeaderLockId: uniqueLockId(), dbTxLockNonce: await randomStr(32, 'base32'), + bskyAppViewProxy: false, ...params, }) diff --git a/packages/pds/tests/proxied/timeline-skeleton.test.ts b/packages/pds/tests/proxied/timeline-skeleton.test.ts index 680dd3423d4..2131150e3a3 100644 --- a/packages/pds/tests/proxied/timeline-skeleton.test.ts +++ b/packages/pds/tests/proxied/timeline-skeleton.test.ts @@ -15,7 +15,8 @@ describe('proxies timeline skeleton', () => { network = await TestNetwork.create({ dbPostgresSchema: 'proxy_timeline_skeleton', pds: { - enableAppView: true, + enableInProcessAppView: true, + bskyAppViewProxy: false, }, }) agent = network.pds.getClient() From 03200c1d8b09d248fa7a9d287cb99db045285dba Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Mon, 3 Jul 2023 21:21:16 +0200 Subject: [PATCH 06/13] :bug: Only ignore reports for specific at-uri when ignoreSubject contains at-uri (#1251) --- .../bsky/src/services/moderation/index.ts | 27 ++++++++++++++--- packages/pds/src/services/moderation/index.ts | 26 ++++++++++++++--- .../admin/get-moderation-reports.test.ts | 29 +++++++++++++++++-- 3 files changed, 72 insertions(+), 10 deletions(-) diff --git a/packages/bsky/src/services/moderation/index.ts b/packages/bsky/src/services/moderation/index.ts index 4d86b4fa9b3..99715866d2a 100644 --- a/packages/bsky/src/services/moderation/index.ts +++ b/packages/bsky/src/services/moderation/index.ts @@ -104,12 +104,31 @@ export class ModerationService { .orWhere('subjectUri', '=', subject) }) } + if (ignoreSubjects?.length) { - builder = builder.where((qb) => { - return qb - .where('subjectDid', 'not in', ignoreSubjects) - .where('subjectUri', 'not in', ignoreSubjects) + const ignoreUris: string[] = [] + const ignoreDids: string[] = [] + + ignoreSubjects.forEach((subject) => { + if (subject.startsWith('at://')) { + ignoreUris.push(subject) + } else if (subject.startsWith('did:')) { + ignoreDids.push(subject) + } }) + + if (ignoreDids.length) { + builder = builder.where('subjectDid', 'not in', ignoreDids) + } + if (ignoreUris.length) { + builder = builder.where((qb) => { + // Without the null condition, postgres will ignore all reports where `subjectUri` is null + // which will make all the account reports be ignored as well + return qb + .where('subjectUri', 'not in', ignoreUris) + .orWhere('subjectUri', 'is', null) + }) + } } if (reporters?.length) { diff --git a/packages/pds/src/services/moderation/index.ts b/packages/pds/src/services/moderation/index.ts index 4a7b3c9eefc..18e3ceb5608 100644 --- a/packages/pds/src/services/moderation/index.ts +++ b/packages/pds/src/services/moderation/index.ts @@ -124,11 +124,29 @@ export class ModerationService { } if (ignoreSubjects?.length) { - builder = builder.where((qb) => { - return qb - .where('subjectDid', 'not in', ignoreSubjects) - .where('subjectUri', 'not in', ignoreSubjects) + const ignoreUris: string[] = [] + const ignoreDids: string[] = [] + + ignoreSubjects.forEach((subject) => { + if (subject.startsWith('at://')) { + ignoreUris.push(subject) + } else if (subject.startsWith('did:')) { + ignoreDids.push(subject) + } }) + + if (ignoreDids.length) { + builder = builder.where('subjectDid', 'not in', ignoreDids) + } + if (ignoreUris.length) { + builder = builder.where((qb) => { + // Without the null condition, postgres will ignore all reports where `subjectUri` is null + // which will make all the account reports be ignored as well + return qb + .where('subjectUri', 'not in', ignoreUris) + .orWhere('subjectUri', 'is', null) + }) + } } if (reporters?.length) { diff --git a/packages/pds/tests/views/admin/get-moderation-reports.test.ts b/packages/pds/tests/views/admin/get-moderation-reports.test.ts index 144a75a9f83..0ef0f92685c 100644 --- a/packages/pds/tests/views/admin/get-moderation-reports.test.ts +++ b/packages/pds/tests/views/admin/get-moderation-reports.test.ts @@ -136,15 +136,40 @@ describe('pds admin get moderation reports view', () => { const ignoreSubjects = getDids(allReports).slice(0, 2) - const filteredReports = + const filteredReportsByDid = await agent.api.com.atproto.admin.getModerationReports( { ignoreSubjects }, { headers: { authorization: adminAuth() } }, ) - getDids(filteredReports).forEach((resultDid) => + // Validate that when ignored by DID, all reports for that DID is ignored + getDids(filteredReportsByDid).forEach((resultDid) => expect(ignoreSubjects).not.toContain(resultDid), ) + + const ignoredAtUriSubjects: string[] = [ + `${ + allReports.data.reports.find(({ subject }) => !!subject.uri)?.subject + ?.uri + }`, + ] + const filteredReportsByAtUri = + await agent.api.com.atproto.admin.getModerationReports( + { + ignoreSubjects: ignoredAtUriSubjects, + }, + { headers: { authorization: adminAuth() } }, + ) + + // Validate that when ignored by at uri, only the reports for that at uri is ignored + expect(filteredReportsByAtUri.data.reports.length).toEqual( + allReports.data.reports.length - 1, + ) + expect( + filteredReportsByAtUri.data.reports + .map(({ subject }) => subject.uri) + .filter(Boolean), + ).not.toContain(ignoredAtUriSubjects[0]) }) it('gets all moderation reports.', async () => { From 8815efd86eb298e5809d848d59735d2ec0947f2a Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Mon, 3 Jul 2023 15:03:04 -0500 Subject: [PATCH 07/13] Move timeline construction to appview (#1274) full switch timeline construction to appview --- packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts b/packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts index 3587c3acc7d..dba1b58ece7 100644 --- a/packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts +++ b/packages/pds/src/app-view/api/app/bsky/feed/getTimeline.ts @@ -26,7 +26,8 @@ export default function (server: Server, ctx: AppContext) { body: res.data, } } - if (ctx.canProxyFeedConstruction(req)) { + + if (ctx.cfg.bskyAppViewEndpoint) { const res = await ctx.appviewAgent.api.app.bsky.unspecced.getTimelineSkeleton( { limit, cursor }, From 24be348dfbda473f40acff62d619bc5e1bb354a4 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Mon, 3 Jul 2023 20:07:20 -0500 Subject: [PATCH 08/13] Better propagate errors on repo streams (#1276) better propgate errors on repo streams --- packages/repo/src/util.ts | 25 ++++++- packages/repo/tests/util.test.ts | 21 ++++++ packages/xrpc-server/src/server.ts | 1 + packages/xrpc-server/tests/responses.test.ts | 77 ++++++++++++++++++++ 4 files changed, 120 insertions(+), 4 deletions(-) create mode 100644 packages/repo/tests/util.test.ts create mode 100644 packages/xrpc-server/tests/responses.test.ts diff --git a/packages/repo/src/util.ts b/packages/repo/src/util.ts index 8ec5239fa4f..42a59f6ea82 100644 --- a/packages/repo/src/util.ts +++ b/packages/repo/src/util.ts @@ -10,6 +10,7 @@ import { check, schema, cidForCbor, + byteIterableToStream, } from '@atproto/common' import { ipldToLex, lexToIpld, LexValue, RepoRecord } from '@atproto/lexicon' @@ -33,6 +34,7 @@ import BlockMap from './block-map' import { MissingBlocksError } from './error' import * as parse from './parse' import { Keypair } from '@atproto/crypto' +import { Readable } from 'stream' export async function* verifyIncomingCarBlocks( car: AsyncIterable, @@ -43,16 +45,31 @@ export async function* verifyIncomingCarBlocks( } } -export const writeCar = ( +// we have to turn the car writer output into a stream in order to properly handle errors +export function writeCarStream( root: CID | null, fn: (car: BlockWriter) => Promise, -): AsyncIterable => { +): Readable { const { writer, out } = root !== null ? CarWriter.create(root) : CarWriter.create() - fn(writer).finally(() => writer.close()) + const stream = byteIterableToStream(out) + fn(writer) + .catch((err) => { + stream.destroy(err) + }) + .finally(() => writer.close()) + return stream +} - return out +export async function* writeCar( + root: CID | null, + fn: (car: BlockWriter) => Promise, +): AsyncIterable { + const stream = writeCarStream(root, fn) + for await (const chunk of stream) { + yield chunk + } } export const blocksToCarStream = ( diff --git a/packages/repo/tests/util.test.ts b/packages/repo/tests/util.test.ts new file mode 100644 index 00000000000..f341cadfea9 --- /dev/null +++ b/packages/repo/tests/util.test.ts @@ -0,0 +1,21 @@ +import { dataToCborBlock, wait } from '@atproto/common' +import { writeCar } from '../src' + +describe('Utils', () => { + describe('writeCar()', () => { + it('propagates errors', async () => { + const iterate = async () => { + const iter = writeCar(null, async (car) => { + await wait(1) + const block = await dataToCborBlock({ test: 1 }) + await car.put(block) + throw new Error('Oops!') + }) + for await (const bytes of iter) { + // no-op + } + } + await expect(iterate).rejects.toThrow('Oops!') + }) + }) +}) diff --git a/packages/xrpc-server/src/server.ts b/packages/xrpc-server/src/server.ts index 2c30ae8a440..f8dc5c82f8c 100644 --- a/packages/xrpc-server/src/server.ts +++ b/packages/xrpc-server/src/server.ts @@ -235,6 +235,7 @@ export class Server { } else if (output?.body instanceof Readable) { res.header('Content-Type', output.encoding) res.status(200) + res.once('error', (err) => res.destroy(err)) forwardStreamErrors(output.body, res) output.body.pipe(res) } else if (output) { diff --git a/packages/xrpc-server/tests/responses.test.ts b/packages/xrpc-server/tests/responses.test.ts new file mode 100644 index 00000000000..0eaccba0633 --- /dev/null +++ b/packages/xrpc-server/tests/responses.test.ts @@ -0,0 +1,77 @@ +import * as http from 'http' +import getPort from 'get-port' +import xrpc, { ServiceClient } from '@atproto/xrpc' +import { byteIterableToStream } from '@atproto/common' +import { createServer, closeServer } from './_util' +import * as xrpcServer from '../src' + +const LEXICONS = [ + { + lexicon: 1, + id: 'io.example.readableStream', + defs: { + main: { + type: 'query', + parameters: { + type: 'params', + properties: { + shouldErr: { type: 'boolean' }, + }, + }, + output: { + encoding: 'application/vnd.ipld.car', + }, + }, + }, + }, +] + +describe('Responses', () => { + let s: http.Server + const server = xrpcServer.createServer(LEXICONS) + server.method( + 'io.example.readableStream', + async (ctx: { params: xrpcServer.Params }) => { + async function* iter(): AsyncIterable { + for (let i = 0; i < 5; i++) { + yield new Uint8Array([i]) + } + if (ctx.params.shouldErr) { + throw new Error('error') + } + } + return { + encoding: 'application/vnd.ipld.car', + body: byteIterableToStream(iter()), + } + }, + ) + xrpc.addLexicons(LEXICONS) + + let client: ServiceClient + let url: string + beforeAll(async () => { + const port = await getPort() + s = await createServer(port, server) + url = `http://localhost:${port}` + client = xrpc.service(url) + }) + afterAll(async () => { + await closeServer(s) + }) + + it('returns readable streams of bytes', async () => { + const res = await client.call('io.example.readableStream', { + shouldErr: false, + }) + const expected = new Uint8Array([0, 1, 2, 3, 4]) + expect(res.data).toEqual(expected) + }) + + it('handles errs on readable streams of bytes', async () => { + const attempt = client.call('io.example.readableStream', { + shouldErr: true, + }) + await expect(attempt).rejects.toThrow() + }) +}) From dd4d91ce6b31031d393fd1334640bc987ae513b8 Mon Sep 17 00:00:00 2001 From: devin ivy Date: Tue, 4 Jul 2023 00:15:13 -0400 Subject: [PATCH 09/13] Log pds sequencer leader stats (#1277) log pds sequencer leader stats --- packages/pds/src/index.ts | 12 +++++++++++- packages/pds/src/sequencer/sequencer-leader.ts | 10 +++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/pds/src/index.ts b/packages/pds/src/index.ts index 3f61323da8f..a56fa893930 100644 --- a/packages/pds/src/index.ts +++ b/packages/pds/src/index.ts @@ -21,7 +21,7 @@ import Database from './db' import { ServerAuth } from './auth' import * as error from './error' import compression from './util/compression' -import { dbLogger, loggerMiddleware } from './logger' +import { dbLogger, loggerMiddleware, seqLogger } from './logger' import { ServerConfig } from './config' import { ServerMailer } from './mailer' import { createServer } from './lexicon' @@ -56,6 +56,7 @@ export class PDS { public server?: http.Server private terminator?: HttpTerminator private dbStatsInterval?: NodeJS.Timer + private sequencerStatsInterval?: NodeJS.Timer constructor(opts: { ctx: AppContext @@ -249,6 +250,14 @@ export class PDS { ) }, 10000) } + this.sequencerStatsInterval = setInterval(() => { + if (this.ctx.sequencerLeader.isLeader) { + seqLogger.info( + { seq: this.ctx.sequencerLeader.peekSeqVal() }, + 'sequencer leader stats', + ) + } + }, 500) appviewConsumers.listen(this.ctx) this.ctx.sequencerLeader.run() await this.ctx.sequencer.start() @@ -267,6 +276,7 @@ export class PDS { await this.ctx.backgroundQueue.destroy() await this.ctx.db.close() clearInterval(this.dbStatsInterval) + clearInterval(this.sequencerStatsInterval) } } diff --git a/packages/pds/src/sequencer/sequencer-leader.ts b/packages/pds/src/sequencer/sequencer-leader.ts index 58eb9aaee5f..c951fda0d25 100644 --- a/packages/pds/src/sequencer/sequencer-leader.ts +++ b/packages/pds/src/sequencer/sequencer-leader.ts @@ -12,7 +12,7 @@ export class SequencerLeader { destroyed = false polling = false queued = false - lastSeq: number + private lastSeq: number constructor(public db: Database, lockId = SEQUENCER_LEADER_ID) { this.leader = new Leader(lockId, this.db) @@ -23,6 +23,14 @@ export class SequencerLeader { return this.lastSeq } + peekSeqVal(): number | undefined { + return this.lastSeq + } + + get isLeader() { + return !!this.leader.session + } + async run() { if (this.db.dialect === 'sqlite') return From 0ceed96b11067244d73ea944bfd2a857dc2f069f Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Tue, 4 Jul 2023 18:40:43 -0500 Subject: [PATCH 10/13] Explicit dns servers (#1281) * add ability to setup explicit dns servers * cleanup * fix * reorder * pr feedback --- packages/identity/src/handle/index.ts | 56 +++++++++++++++++++++++---- packages/identity/src/types.ts | 2 + packages/pds/src/config.ts | 10 +++++ packages/pds/src/index.ts | 6 ++- 4 files changed, 66 insertions(+), 8 deletions(-) diff --git a/packages/identity/src/handle/index.ts b/packages/identity/src/handle/index.ts index fa9c016cd99..8780437efa7 100644 --- a/packages/identity/src/handle/index.ts +++ b/packages/identity/src/handle/index.ts @@ -6,9 +6,12 @@ const PREFIX = 'did=' export class HandleResolver { public timeout: number + private backupNameservers: string[] | undefined + private backupNameserverIps: string[] | undefined constructor(opts: HandleResolverOpts = {}) { this.timeout = opts.timeout ?? 3000 + this.backupNameservers = opts.backupNameservers } async resolve(handle: string): Promise { @@ -23,7 +26,11 @@ export class HandleResolver { httpAbort.abort() return dnsRes } - return httpPromise + const res = await httpPromise + if (res) { + return res + } + return this.resolveDnsBackup(handle) } async resolveDns(handle: string): Promise { @@ -33,12 +40,7 @@ export class HandleResolver { } catch (err) { return undefined } - const results = chunkedResults.map((chunks) => chunks.join('')) - const found = results.filter((i) => i.startsWith(PREFIX)) - if (found.length !== 1) { - return undefined - } - return found[0].slice(PREFIX.length) + return this.parseDnsResult(chunkedResults) } async resolveHttp( @@ -57,4 +59,44 @@ export class HandleResolver { return undefined } } + + async resolveDnsBackup(handle: string): Promise { + let chunkedResults: string[][] + try { + const backupIps = await this.getBackupNameserverIps() + if (!backupIps || backupIps.length < 1) return undefined + const resolver = new dns.Resolver() + resolver.setServers(backupIps) + chunkedResults = await resolver.resolveTxt(`${SUBDOMAIN}.${handle}`) + } catch (err) { + return undefined + } + return this.parseDnsResult(chunkedResults) + } + + parseDnsResult(chunkedResults: string[][]): string | undefined { + const results = chunkedResults.map((chunks) => chunks.join('')) + const found = results.filter((i) => i.startsWith(PREFIX)) + if (found.length !== 1) { + return undefined + } + return found[0].slice(PREFIX.length) + } + + private async getBackupNameserverIps(): Promise { + if (!this.backupNameservers) { + return undefined + } else if (!this.backupNameserverIps) { + const responses = await Promise.allSettled( + this.backupNameservers.map((h) => dns.lookup(h)), + ) + for (const res of responses) { + if (res.status === 'fulfilled') { + this.backupNameserverIps ??= [] + this.backupNameserverIps.push(res.value.address) + } + } + } + return this.backupNameserverIps + } } diff --git a/packages/identity/src/types.ts b/packages/identity/src/types.ts index fdb1857f3b5..f1d983e6742 100644 --- a/packages/identity/src/types.ts +++ b/packages/identity/src/types.ts @@ -4,10 +4,12 @@ export type IdentityResolverOpts = { timeout?: number plcUrl?: string didCache?: DidCache + backupNameservers?: string[] } export type HandleResolverOpts = { timeout?: number + backupNameservers?: string[] } export type DidResolverOpts = { diff --git a/packages/pds/src/config.ts b/packages/pds/src/config.ts index b860cb75908..282f7929a10 100644 --- a/packages/pds/src/config.ts +++ b/packages/pds/src/config.ts @@ -35,6 +35,7 @@ export interface ServerConfigValues { databaseLocation?: string availableUserDomains: string[] + handleResolveNameservers?: string[] imgUriSalt: string imgUriKey: string @@ -136,6 +137,10 @@ export class ServerConfig { ? process.env.AVAILABLE_USER_DOMAINS.split(',') : [] + const handleResolveNameservers = process.env.HANDLE_RESOLVE_NAMESERVERS + ? process.env.HANDLE_RESOLVE_NAMESERVERS.split(',') + : [] + const imgUriSalt = process.env.IMG_URI_SALT || '9dd04221f5755bce5f55f47464c27e1e' const imgUriKey = @@ -215,6 +220,7 @@ export class ServerConfig { termsOfServiceUrl, databaseLocation, availableUserDomains, + handleResolveNameservers, imgUriSalt, imgUriKey, imgUriEndpoint, @@ -368,6 +374,10 @@ export class ServerConfig { return this.cfg.availableUserDomains } + get handleResolveNameservers() { + return this.cfg.handleResolveNameservers + } + get imgUriSalt() { return this.cfg.imgUriSalt } diff --git a/packages/pds/src/index.ts b/packages/pds/src/index.ts index a56fa893930..f77b654a069 100644 --- a/packages/pds/src/index.ts +++ b/packages/pds/src/index.ts @@ -96,7 +96,11 @@ export class PDS { config.didCacheStaleTTL, config.didCacheMaxTTL, ) - const idResolver = new IdResolver({ plcUrl: config.didPlcUrl, didCache }) + const idResolver = new IdResolver({ + plcUrl: config.didPlcUrl, + didCache, + backupNameservers: config.handleResolveNameservers, + }) const messageDispatcher = new MessageDispatcher() const sequencer = new Sequencer(db) From 4f7fd8b1180e07f4d8531ad0aab1e185392b2dc4 Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Tue, 4 Jul 2023 20:58:06 -0500 Subject: [PATCH 11/13] Thread through id-resolver cfg (#1282) thread through id-resolver-cfg --- packages/identity/src/id-resolver.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/identity/src/id-resolver.ts b/packages/identity/src/id-resolver.ts index e400e201832..ccf42ca9574 100644 --- a/packages/identity/src/id-resolver.ts +++ b/packages/identity/src/id-resolver.ts @@ -8,7 +8,10 @@ export class IdResolver { constructor(opts: IdentityResolverOpts = {}) { const { timeout = 3000, plcUrl, didCache } = opts - this.handle = new HandleResolver({ timeout }) + this.handle = new HandleResolver({ + timeout, + backupNameservers: opts.backupNameservers, + }) this.did = new DidResolver({ timeout, plcUrl, didCache }) } } From bb2848e7f5f588c457a8ed117ed1fa7f72af51bf Mon Sep 17 00:00:00 2001 From: devin ivy Date: Wed, 5 Jul 2023 15:01:13 -0400 Subject: [PATCH 12/13] Bsky log commit error (#1275) * don't bail on bad record index * add build * temporarily disable check, full reindex on rabase * don't bail on bad record index during rebase, track last commit on rebase * log bsky repo subscription stats * add running and waiting count to repo sub stats * re-enable fast path for happy rebases * only hold onto seq in cursor consecutivelist, don't hold onto whole completed messages --- .../workflows/build-and-push-bsky-aws.yaml | 1 + packages/bsky/src/index.ts | 17 +++++++- packages/bsky/src/services/indexing/index.ts | 5 ++- packages/bsky/src/subscription/repo.ts | 42 ++++++++++++++----- 4 files changed, 53 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-and-push-bsky-aws.yaml b/.github/workflows/build-and-push-bsky-aws.yaml index b656dec89c9..9e5c2e9b870 100644 --- a/.github/workflows/build-and-push-bsky-aws.yaml +++ b/.github/workflows/build-and-push-bsky-aws.yaml @@ -3,6 +3,7 @@ on: push: branches: - main + - bsky-log-commit-error env: REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }} USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }} diff --git a/packages/bsky/src/index.ts b/packages/bsky/src/index.ts index 771b0ef27c3..8af95aba95c 100644 --- a/packages/bsky/src/index.ts +++ b/packages/bsky/src/index.ts @@ -9,7 +9,7 @@ import { IdResolver } from '@atproto/identity' import API, { health, blobResolver } from './api' import Database from './db' import * as error from './error' -import { dbLogger, loggerMiddleware } from './logger' +import { dbLogger, loggerMiddleware, subLogger } from './logger' import { ServerConfig } from './config' import { createServer } from './lexicon' import { ImageUriBuilder } from './image/uri' @@ -41,6 +41,7 @@ export class BskyAppView { public server?: http.Server private terminator?: HttpTerminator private dbStatsInterval: NodeJS.Timer + private subStatsInterval: NodeJS.Timer constructor(opts: { ctx: AppContext @@ -188,6 +189,19 @@ export class BskyAppView { 'background queue stats', ) }, 10000) + if (this.sub) { + this.subStatsInterval = setInterval(() => { + subLogger.info( + { + seq: this.sub?.lastSeq, + cursor: this.sub?.lastCursor, + runningCount: this.sub?.repoQueue.main.pending, + waitingCount: this.sub?.repoQueue.main.size, + }, + 'repo subscription stats', + ) + }, 500) + } const server = this.app.listen(this.ctx.cfg.port) this.server = server this.terminator = createHttpTerminator({ server }) @@ -201,6 +215,7 @@ export class BskyAppView { async destroy(): Promise { await this.ctx.didCache.destroy() await this.sub?.destroy() + clearInterval(this.subStatsInterval) await this.terminator?.terminate() await this.ctx.backgroundQueue.destroy() await this.ctx.db.close() diff --git a/packages/bsky/src/services/indexing/index.ts b/packages/bsky/src/services/indexing/index.ts index da79cbb6345..25ecb8ccebe 100644 --- a/packages/bsky/src/services/indexing/index.ts +++ b/packages/bsky/src/services/indexing/index.ts @@ -203,7 +203,10 @@ export class IndexingService { 'skipping indexing of invalid record', ) } else { - throw err + subLogger.error( + { err, did, commit, uri: uri.toString(), cid: cid.toString() }, + 'skipping indexing due to error processing record', + ) } } }) diff --git a/packages/bsky/src/subscription/repo.ts b/packages/bsky/src/subscription/repo.ts index 2f4047805d1..db0f84dca8f 100644 --- a/packages/bsky/src/subscription/repo.ts +++ b/packages/bsky/src/subscription/repo.ts @@ -27,8 +27,10 @@ export class RepoSubscription { leader = new Leader(this.subLockId, this.ctx.db) repoQueue: PartitionedQueue cursorQueue = new LatestQueue() - consecutive = new ConsecutiveList() + consecutive = new ConsecutiveList() destroyed = false + lastSeq: number | undefined + lastCursor: number | undefined constructor( public ctx: AppContext, @@ -56,7 +58,8 @@ export class RepoSubscription { ) continue } - const item = this.consecutive.push(details.message) + this.lastSeq = details.seq + const item = this.consecutive.push(details.seq) this.repoQueue .add(details.repo, () => this.handleMessage(details.message)) .catch((err) => { @@ -113,7 +116,7 @@ export class RepoSubscription { this.destroyed = false this.repoQueue = new PartitionedQueue({ concurrency: this.concurrency }) this.cursorQueue = new LatestQueue() - this.consecutive = new ConsecutiveList() + this.consecutive = new ConsecutiveList() await this.run() } @@ -138,14 +141,19 @@ export class RepoSubscription { const indexRecords = async () => { const { root, rootCid, ops } = await getOps(msg) if (msg.tooBig) { - return await indexingService.indexRepo(msg.repo, rootCid.toString()) + await indexingService.indexRepo(msg.repo, rootCid.toString()) + await indexingService.setCommitLastSeen(root, msg) + return } if (msg.rebase) { const needsReindex = await indexingService.checkCommitNeedsIndexing( root, ) - if (!needsReindex) return - return await indexingService.indexRepo(msg.repo, rootCid.toString()) + if (needsReindex) { + await indexingService.indexRepo(msg.repo, rootCid.toString()) + } + await indexingService.setCommitLastSeen(root, msg) + return } for (const op of ops) { if (op.action === WriteOpAction.Delete) { @@ -171,7 +179,16 @@ export class RepoSubscription { 'skipping indexing of invalid record', ) } else { - throw err + subLogger.error( + { + err, + did: msg.repo, + commit: msg.commit.toString(), + uri: op.uri.toString(), + cid: op.cid.toString(), + }, + 'skipping indexing due to error processing record', + ) } } } @@ -195,10 +212,10 @@ export class RepoSubscription { await services.indexing(db).tombstoneActor(msg.did) } - private async handleCursor(msg: ProcessableMessage) { + private async handleCursor(seq: number) { const { db } = this.ctx await db.transaction(async (tx) => { - await this.setState(tx, { cursor: msg.seq }) + await this.setState(tx, { cursor: seq }) }) } @@ -209,7 +226,9 @@ export class RepoSubscription { .where('service', '=', this.service) .where('method', '=', METHOD) .executeTakeFirst() - return sub ? (JSON.parse(sub.state) as State) : { cursor: 0 } + const state = sub ? (JSON.parse(sub.state) as State) : { cursor: 0 } + this.lastCursor = state.cursor + return state } async resetState(): Promise { @@ -222,6 +241,9 @@ export class RepoSubscription { private async setState(tx: Database, state: State): Promise { tx.assertTransaction() + tx.onCommit(() => { + this.lastCursor = state.cursor + }) const res = await tx.db .updateTable('subscription') .where('service', '=', this.service) From 4d1f8d32899b765e17af219329e2e3166b43e54d Mon Sep 17 00:00:00 2001 From: Daniel Holmgren Date: Wed, 5 Jul 2023 16:41:08 -0500 Subject: [PATCH 13/13] Misc scaling (#1284) * limit backsearch to 1 day instead of 3 * lower like count threshold * bump to 6 * disable like count check * disable with friends * preemptively cache last commit * inline list mutes * actor service * label cache * placehodler on popular with friends * bulk sequence * no limit but chunk * bump chunk to 5k * try 10k * fix notify * tweaking * syntax * one more fix * increase backfill allowance * full refresh label cache * limit 1 on mute list * reserve aclu handle * clean up testing with label cache * note on with-friends * rm defer from label cache * label cache error handling * rm branch build --- packages/dev-env/src/bsky.ts | 4 + packages/dev-env/src/network-no-appview.ts | 4 + packages/dev-env/src/network.ts | 6 +- packages/dev-env/src/pds.ts | 8 ++ packages/identifier/src/reserved.ts | 1 + .../src/app-view/api/app/bsky/util/feed.ts | 2 +- .../pds/src/app-view/services/actor/index.ts | 13 ++- .../pds/src/app-view/services/actor/views.ts | 103 ++++++++++-------- .../pds/src/app-view/services/feed/index.ts | 43 ++++++-- .../pds/src/app-view/services/graph/index.ts | 16 +++ .../pds/src/app-view/services/label/index.ts | 37 +++++-- packages/pds/src/context.ts | 6 + packages/pds/src/feed-gen/with-friends.ts | 73 ++++++++----- packages/pds/src/index.ts | 7 ++ packages/pds/src/label-cache.ts | 90 +++++++++++++++ packages/pds/src/sequencer/outbox.ts | 4 +- .../pds/src/sequencer/sequencer-leader.ts | 22 ++-- packages/pds/src/sequencer/sequencer.ts | 2 +- packages/pds/src/services/index.ts | 9 +- packages/pds/src/services/repo/index.ts | 8 +- packages/pds/src/sql-repo-storage.ts | 18 +++ packages/pds/tests/_util.ts | 8 ++ packages/pds/tests/account-deletion.test.ts | 2 +- packages/pds/tests/algos/hot-classic.test.ts | 4 +- packages/pds/tests/algos/whats-hot.test.ts | 4 +- packages/pds/tests/algos/with-friends.test.ts | 2 +- packages/pds/tests/blob-deletes.test.ts | 8 +- packages/pds/tests/event-stream/sync.test.ts | 1 + packages/pds/tests/feed-generation.test.ts | 2 +- packages/pds/tests/indexing.test.ts | 12 +- packages/pds/tests/labeler/labeler.test.ts | 8 +- .../pds/tests/migrations/blob-creator.test.ts | 2 +- .../migrations/indexed-at-on-record.test.ts | 2 +- .../tests/migrations/post-hierarchy.test.ts | 2 +- .../migrations/repo-sync-data-pt2.test.ts | 2 +- .../tests/migrations/repo-sync-data.test.ts | 2 +- .../migrations/user-partitioned-cids.test.ts | 2 +- .../migrations/user-table-did-pkey.test.ts | 2 +- packages/pds/tests/moderation.test.ts | 2 +- packages/pds/tests/proxied/feedgen.test.ts | 1 - packages/pds/tests/views/actor-search.test.ts | 2 +- packages/pds/tests/views/author-feed.test.ts | 2 +- packages/pds/tests/views/blocks.test.ts | 2 +- packages/pds/tests/views/follows.test.ts | 2 +- packages/pds/tests/views/likes.test.ts | 2 +- packages/pds/tests/views/mute-lists.test.ts | 2 +- packages/pds/tests/views/mutes.test.ts | 2 +- .../pds/tests/views/notifications.test.ts | 4 +- packages/pds/tests/views/popular.test.ts | 4 +- packages/pds/tests/views/posts.test.ts | 2 +- packages/pds/tests/views/profile.test.ts | 2 +- packages/pds/tests/views/reposts.test.ts | 2 +- packages/pds/tests/views/suggestions.test.ts | 2 +- packages/pds/tests/views/thread.test.ts | 7 +- packages/pds/tests/views/timeline.test.ts | 2 +- 55 files changed, 414 insertions(+), 169 deletions(-) create mode 100644 packages/pds/src/label-cache.ts diff --git a/packages/dev-env/src/bsky.ts b/packages/dev-env/src/bsky.ts index 75206fe768c..da3010adf22 100644 --- a/packages/dev-env/src/bsky.ts +++ b/packages/dev-env/src/bsky.ts @@ -86,6 +86,10 @@ export class TestBsky { return new AtpAgent({ service: this.url }) } + async processAll() { + await this.ctx.backgroundQueue.processAll() + } + async close() { await this.server.destroy() } diff --git a/packages/dev-env/src/network-no-appview.ts b/packages/dev-env/src/network-no-appview.ts index f5244eff2db..24a0b72fb59 100644 --- a/packages/dev-env/src/network-no-appview.ts +++ b/packages/dev-env/src/network-no-appview.ts @@ -37,6 +37,10 @@ export class TestNetworkNoAppView { return fg } + async processAll() { + await this.pds.processAll() + } + async close() { await Promise.all(this.feedGens.map((fg) => fg.close())) await this.pds.close() diff --git a/packages/dev-env/src/network.ts b/packages/dev-env/src/network.ts index 5d589335346..1cce66cfde9 100644 --- a/packages/dev-env/src/network.ts +++ b/packages/dev-env/src/network.ts @@ -54,7 +54,6 @@ export class TestNetwork extends TestNetworkNoAppView { } async processFullSubscription(timeout = 5000) { - if (!this.bsky) return const sub = this.bsky.sub if (!sub) return const { db } = this.pds.ctx.db @@ -76,10 +75,9 @@ export class TestNetwork extends TestNetworkNoAppView { } async processAll(timeout?: number) { - await this.pds.ctx.backgroundQueue.processAll() - if (!this.bsky) return + await this.pds.processAll() await this.processFullSubscription(timeout) - await this.bsky.ctx.backgroundQueue.processAll() + await this.bsky.processAll() } async serviceHeaders(did: string, aud?: string) { diff --git a/packages/dev-env/src/pds.ts b/packages/dev-env/src/pds.ts index ee0c307f141..28b6727c4d8 100644 --- a/packages/dev-env/src/pds.ts +++ b/packages/dev-env/src/pds.ts @@ -96,6 +96,9 @@ export class TestPds { }) await server.start() + + // we refresh label cache by hand in `processAll` instead of on a timer + server.ctx.labelCache.stop() return new TestPds(url, port, server) } @@ -123,6 +126,11 @@ export class TestPds { } } + async processAll() { + await this.ctx.backgroundQueue.processAll() + await this.ctx.labelCache.fullRefresh() + } + async close() { await this.server.destroy() } diff --git a/packages/identifier/src/reserved.ts b/packages/identifier/src/reserved.ts index 83180160e61..c49c85f5378 100644 --- a/packages/identifier/src/reserved.ts +++ b/packages/identifier/src/reserved.ts @@ -864,6 +864,7 @@ const famousAccounts = [ // reserving some large twitter accounts (top 100 by followers according to wikidata dump) '10ronaldinho', '3gerardpique', + 'aclu', 'adele', 'akshaykumar', 'aliaa08', diff --git a/packages/pds/src/app-view/api/app/bsky/util/feed.ts b/packages/pds/src/app-view/api/app/bsky/util/feed.ts index 606580b3046..9d9d0323b95 100644 --- a/packages/pds/src/app-view/api/app/bsky/util/feed.ts +++ b/packages/pds/src/app-view/api/app/bsky/util/feed.ts @@ -12,7 +12,7 @@ export class FeedKeyset extends TimeCidKeyset { } // For users with sparse feeds, avoid scanning more than one week for a single page -export const getFeedDateThreshold = (from: string | undefined, days = 3) => { +export const getFeedDateThreshold = (from: string | undefined, days = 1) => { const timelineDateThreshold = from ? new Date(from) : new Date() timelineDateThreshold.setDate(timelineDateThreshold.getDate() - days) return timelineDateThreshold.toISOString() diff --git a/packages/pds/src/app-view/services/actor/index.ts b/packages/pds/src/app-view/services/actor/index.ts index 8ed28e4691b..e6c6fd6e756 100644 --- a/packages/pds/src/app-view/services/actor/index.ts +++ b/packages/pds/src/app-view/services/actor/index.ts @@ -3,15 +3,20 @@ import { DidHandle } from '../../../db/tables/did-handle' import { notSoftDeletedClause } from '../../../db/util' import { ActorViews } from './views' import { ImageUriBuilder } from '../../../image/uri' +import { LabelCache } from '../../../label-cache' export class ActorService { - constructor(public db: Database, public imgUriBuilder: ImageUriBuilder) {} + constructor( + public db: Database, + public imgUriBuilder: ImageUriBuilder, + public labelCache: LabelCache, + ) {} - static creator(imgUriBuilder: ImageUriBuilder) { - return (db: Database) => new ActorService(db, imgUriBuilder) + static creator(imgUriBuilder: ImageUriBuilder, labelCache: LabelCache) { + return (db: Database) => new ActorService(db, imgUriBuilder, labelCache) } - views = new ActorViews(this.db, this.imgUriBuilder) + views = new ActorViews(this.db, this.imgUriBuilder, this.labelCache) async getActor( handleOrDid: string, diff --git a/packages/pds/src/app-view/services/actor/views.ts b/packages/pds/src/app-view/services/actor/views.ts index 50fd021a78c..b0f652ef901 100644 --- a/packages/pds/src/app-view/services/actor/views.ts +++ b/packages/pds/src/app-view/services/actor/views.ts @@ -8,13 +8,19 @@ import { DidHandle } from '../../../db/tables/did-handle' import Database from '../../../db' import { ImageUriBuilder } from '../../../image/uri' import { LabelService } from '../label' -import { ListViewBasic } from '../../../lexicon/types/app/bsky/graph/defs' +import { GraphService } from '../graph' +import { LabelCache } from '../../../label-cache' export class ActorViews { - constructor(private db: Database, private imgUriBuilder: ImageUriBuilder) {} + constructor( + private db: Database, + private imgUriBuilder: ImageUriBuilder, + private labelCache: LabelCache, + ) {} services = { - label: LabelService.creator(), + label: LabelService.creator(this.labelCache)(this.db), + graph: GraphService.creator(this.imgUriBuilder)(this.db), } profileDetailed( @@ -82,18 +88,30 @@ export class ActorViews { .where('mutedByDid', '=', viewer) .select('did') .as('requesterMuted'), + this.db.db + .selectFrom('list_item') + .innerJoin('list_mute', 'list_mute.listUri', 'list_item.listUri') + .where('list_mute.mutedByDid', '=', viewer) + .whereRef('list_item.subjectDid', '=', ref('did_handle.did')) + .select('list_item.listUri') + .limit(1) + .as('requesterMutedByList'), ]) - const [profileInfos, labels, listMutes] = await Promise.all([ + const [profileInfos, labels] = await Promise.all([ profileInfosQb.execute(), - this.services.label(this.db).getLabelsForSubjects(dids), - this.getListMutes(dids, viewer), + this.services.label.getLabelsForSubjects(dids), ]) const profileInfoByDid = profileInfos.reduce((acc, info) => { return Object.assign(acc, { [info.did]: info }) }, {} as Record>) + const listUris: string[] = profileInfos + .map((a) => a.requesterMutedByList) + .filter((list) => !!list) + const listViews = await this.services.graph.getListViews(listUris, viewer) + const views = results.map((result) => { const profileInfo = profileInfoByDid[result.did] const avatar = profileInfo?.avatarCid @@ -114,8 +132,14 @@ export class ActorViews { postsCount: profileInfo?.postsCount || 0, indexedAt: profileInfo?.indexedAt || undefined, viewer: { - muted: !!profileInfo?.requesterMuted || !!listMutes[result.did], - mutedByList: listMutes[result.did], + muted: + !!profileInfo?.requesterMuted || + !!profileInfo?.requesterMutedByList, + mutedByList: profileInfo.requesterMutedByList + ? this.services.graph.formatListViewBasic( + listViews[profileInfo.requesterMutedByList], + ) + : undefined, blockedBy: !!profileInfo.requesterBlockedBy, blocking: profileInfo.requesterBlocking || undefined, following: profileInfo?.requesterFollowing || undefined, @@ -181,18 +205,30 @@ export class ActorViews { .where('mutedByDid', '=', viewer) .select('did') .as('requesterMuted'), + this.db.db + .selectFrom('list_item') + .innerJoin('list_mute', 'list_mute.listUri', 'list_item.listUri') + .where('list_mute.mutedByDid', '=', viewer) + .whereRef('list_item.subjectDid', '=', ref('did_handle.did')) + .select('list_item.listUri') + .limit(1) + .as('requesterMutedByList'), ]) - const [profileInfos, labels, listMutes] = await Promise.all([ + const [profileInfos, labels] = await Promise.all([ profileInfosQb.execute(), - this.services.label(this.db).getLabelsForSubjects(dids), - this.getListMutes(dids, viewer), + this.services.label.getLabelsForSubjects(dids), ]) const profileInfoByDid = profileInfos.reduce((acc, info) => { return Object.assign(acc, { [info.did]: info }) }, {} as Record>) + const listUris: string[] = profileInfos + .map((a) => a.requesterMutedByList) + .filter((list) => !!list) + const listViews = await this.services.graph.getListViews(listUris, viewer) + const views = results.map((result) => { const profileInfo = profileInfoByDid[result.did] const avatar = profileInfo?.avatarCid @@ -206,8 +242,14 @@ export class ActorViews { avatar, indexedAt: profileInfo?.indexedAt || undefined, viewer: { - muted: !!profileInfo?.requesterMuted || !!listMutes[result.did], - mutedByList: listMutes[result.did], + muted: + !!profileInfo?.requesterMuted || + !!profileInfo?.requesterMutedByList, + mutedByList: profileInfo.requesterMutedByList + ? this.services.graph.formatListViewBasic( + listViews[profileInfo.requesterMutedByList], + ) + : undefined, blockedBy: !!profileInfo.requesterBlockedBy, blocking: profileInfo.requesterBlocking || undefined, following: profileInfo?.requesterFollowing || undefined, @@ -245,41 +287,6 @@ export class ActorViews { return Array.isArray(result) ? views : views[0] } - - async getListMutes( - subjects: string[], - mutedBy: string, - ): Promise> { - if (subjects.length < 1) return {} - const res = await this.db.db - .selectFrom('list_item') - .innerJoin('list_mute', 'list_mute.listUri', 'list_item.listUri') - .innerJoin('list', 'list.uri', 'list_item.listUri') - .where('list_mute.mutedByDid', '=', mutedBy) - .where('list_item.subjectDid', 'in', subjects) - .selectAll('list') - .select('list_item.subjectDid as subjectDid') - .execute() - return res.reduce( - (acc, cur) => ({ - ...acc, - [cur.subjectDid]: { - uri: cur.uri, - cid: cur.cid, - name: cur.name, - purpose: cur.purpose, - avatar: cur.avatarCid - ? this.imgUriBuilder.getCommonSignedUri('avatar', cur.avatarCid) - : undefined, - viewer: { - muted: true, - }, - indexedAt: cur.indexedAt, - }, - }), - {} as Record, - ) - } } type ActorResult = DidHandle diff --git a/packages/pds/src/app-view/services/feed/index.ts b/packages/pds/src/app-view/services/feed/index.ts index 326588efe36..732ba6623b3 100644 --- a/packages/pds/src/app-view/services/feed/index.ts +++ b/packages/pds/src/app-view/services/feed/index.ts @@ -1,6 +1,7 @@ import { sql } from 'kysely' import { AtUri } from '@atproto/uri' import { dedupeStrs } from '@atproto/common' +import { cborToLexRecord } from '@atproto/repo' import Database from '../../../db' import { countAll, notSoftDeletedClause } from '../../../db/util' import { ImageUriBuilder } from '../../../image/uri' @@ -28,21 +29,25 @@ import { LabelService, Labels } from '../label' import { ActorService } from '../actor' import { GraphService } from '../graph' import { FeedViews } from './views' -import { cborToLexRecord } from '@atproto/repo' +import { LabelCache } from '../../../label-cache' export * from './types' export class FeedService { - constructor(public db: Database, public imgUriBuilder: ImageUriBuilder) {} + constructor( + public db: Database, + public imgUriBuilder: ImageUriBuilder, + public labelCache: LabelCache, + ) {} - static creator(imgUriBuilder: ImageUriBuilder) { - return (db: Database) => new FeedService(db, imgUriBuilder) + static creator(imgUriBuilder: ImageUriBuilder, labelCache: LabelCache) { + return (db: Database) => new FeedService(db, imgUriBuilder, labelCache) } views = new FeedViews(this.db, this.imgUriBuilder) services = { - label: LabelService.creator()(this.db), - actor: ActorService.creator(this.imgUriBuilder)(this.db), + label: LabelService.creator(this.labelCache)(this.db), + actor: ActorService.creator(this.imgUriBuilder, this.labelCache)(this.db), graph: GraphService.creator(this.imgUriBuilder)(this.db), } @@ -114,7 +119,7 @@ export class FeedService { if (dids.length < 1) return {} const { ref } = this.db.db.dynamic const { skipLabels = false, includeSoftDeleted = false } = opts ?? {} - const [actors, labels, listMutes] = await Promise.all([ + const [actors, labels] = await Promise.all([ this.db.db .selectFrom('did_handle') .where('did_handle.did', 'in', dids) @@ -160,11 +165,25 @@ export class FeedService { .where('mutedByDid', '=', requester) .select('did') .as('requesterMuted'), + this.db.db + .selectFrom('list_item') + .innerJoin('list_mute', 'list_mute.listUri', 'list_item.listUri') + .where('list_mute.mutedByDid', '=', requester) + .whereRef('list_item.subjectDid', '=', ref('did_handle.did')) + .select('list_item.listUri') + .limit(1) + .as('requesterMutedByList'), ]) .execute(), this.services.label.getLabelsForSubjects(skipLabels ? [] : dids), - this.services.actor.views.getListMutes(dids, requester), ]) + const listUris: string[] = actors + .map((a) => a.requesterMutedByList) + .filter((list) => !!list) + const listViews = await this.services.graph.getListViews( + listUris, + requester, + ) return actors.reduce((acc, cur) => { const actorLabels = labels[cur.did] ?? [] return { @@ -177,8 +196,12 @@ export class FeedService { ? this.imgUriBuilder.getCommonSignedUri('avatar', cur.avatarCid) : undefined, viewer: { - muted: !!cur?.requesterMuted || !!listMutes[cur.did], - mutedByList: listMutes[cur.did], + muted: !!cur?.requesterMuted || !!cur?.requesterMutedByList, + mutedByList: cur.requesterMutedByList + ? this.services.graph.formatListViewBasic( + listViews[cur.requesterMutedByList], + ) + : undefined, blockedBy: !!cur?.requesterBlockedBy, blocking: cur?.requesterBlocking || undefined, following: cur?.requesterFollowing || undefined, diff --git a/packages/pds/src/app-view/services/graph/index.ts b/packages/pds/src/app-view/services/graph/index.ts index bded56aeed4..94f7c58f54d 100644 --- a/packages/pds/src/app-view/services/graph/index.ts +++ b/packages/pds/src/app-view/services/graph/index.ts @@ -132,6 +132,22 @@ export class GraphService { }, } } + + formatListViewBasic(list: ListInfo) { + return { + uri: list.uri, + cid: list.cid, + name: list.name, + purpose: list.purpose, + avatar: list.avatarCid + ? this.imgUriBuilder.getCommonSignedUri('avatar', list.avatarCid) + : undefined, + indexedAt: list.indexedAt, + viewer: { + muted: !!list.viewerMuted, + }, + } + } } type ListInfo = List & { diff --git a/packages/pds/src/app-view/services/label/index.ts b/packages/pds/src/app-view/services/label/index.ts index 9de4670c16e..aa00ca29f95 100644 --- a/packages/pds/src/app-view/services/label/index.ts +++ b/packages/pds/src/app-view/services/label/index.ts @@ -3,14 +3,15 @@ import Database from '../../../db' import { Label } from '../../../lexicon/types/com/atproto/label/defs' import { ids } from '../../../lexicon/lexicons' import { sql } from 'kysely' +import { LabelCache } from '../../../label-cache' export type Labels = Record export class LabelService { - constructor(public db: Database) {} + constructor(public db: Database, public cache: LabelCache) {} - static creator() { - return (db: Database) => new LabelService(db) + static creator(cache: LabelCache) { + return (db: Database) => new LabelService(db, cache) } async formatAndCreate( @@ -63,14 +64,17 @@ export class LabelService { async getLabelsForUris( subjects: string[], includeNeg?: boolean, + skipCache?: boolean, ): Promise { if (subjects.length < 1) return {} - const res = await this.db.db - .selectFrom('label') - .where('label.uri', 'in', subjects) - .if(!includeNeg, (qb) => qb.where('neg', '=', 0)) - .selectAll() - .execute() + const res = skipCache + ? await this.db.db + .selectFrom('label') + .where('label.uri', 'in', subjects) + .if(!includeNeg, (qb) => qb.where('neg', '=', 0)) + .selectAll() + .execute() + : this.cache.forSubjects(subjects, includeNeg) return res.reduce((acc, cur) => { acc[cur.uri] ??= [] acc[cur.uri].push({ @@ -86,6 +90,7 @@ export class LabelService { async getLabelsForSubjects( subjects: string[], includeNeg?: boolean, + skipCache?: boolean, ): Promise { if (subjects.length < 1) return {} const expandedSubjects = subjects.flatMap((subject) => { @@ -97,7 +102,11 @@ export class LabelService { } return subject }) - const labels = await this.getLabelsForUris(expandedSubjects, includeNeg) + const labels = await this.getLabelsForUris( + expandedSubjects, + includeNeg, + skipCache, + ) return Object.keys(labels).reduce((acc, cur) => { const uri = cur.startsWith('at://') ? new AtUri(cur) : null if ( @@ -116,8 +125,12 @@ export class LabelService { }, {} as Labels) } - async getLabels(subject: string, includeNeg?: boolean): Promise { - const labels = await this.getLabelsForUris([subject], includeNeg) + async getLabels( + subject: string, + includeNeg?: boolean, + skipCache?: boolean, + ): Promise { + const labels = await this.getLabelsForUris([subject], includeNeg, skipCache) return labels[subject] ?? [] } diff --git a/packages/pds/src/context.ts b/packages/pds/src/context.ts index 1e3ce8167d4..093edb1ba84 100644 --- a/packages/pds/src/context.ts +++ b/packages/pds/src/context.ts @@ -18,6 +18,7 @@ import { BackgroundQueue } from './event-stream/background-queue' import DidSqlCache from './did-cache' import { MountedAlgos } from './feed-gen/types' import { Crawlers } from './crawlers' +import { LabelCache } from './label-cache' export class AppContext { private _appviewAgent: AtpAgent | null @@ -39,6 +40,7 @@ export class AppContext { sequencer: Sequencer sequencerLeader: SequencerLeader labeler: Labeler + labelCache: LabelCache backgroundQueue: BackgroundQueue crawlers: Crawlers algos: MountedAlgos @@ -131,6 +133,10 @@ export class AppContext { return this.opts.labeler } + get labelCache(): LabelCache { + return this.opts.labelCache + } + get backgroundQueue(): BackgroundQueue { return this.opts.backgroundQueue } diff --git a/packages/pds/src/feed-gen/with-friends.ts b/packages/pds/src/feed-gen/with-friends.ts index a4a40364563..682387a5e61 100644 --- a/packages/pds/src/feed-gen/with-friends.ts +++ b/packages/pds/src/feed-gen/with-friends.ts @@ -12,39 +12,56 @@ const handler: AlgoHandler = async ( params: SkeletonParams, requester: string, ): Promise => { - const { cursor, limit = 50 } = params - const accountService = ctx.services.account(ctx.db) - const feedService = ctx.services.appView.feed(ctx.db) - const graphService = ctx.services.appView.graph(ctx.db) + // Temporary change to only return a post notifying users that the feed is down + return { + feedItems: [ + { + type: 'post', + uri: 'at://did:plc:z72i7hdynmk6r22z27h6tvur/app.bsky.feed.post/3jzinucnmbi2c', + cid: 'bafyreifmtn55tubbv7tefrq277nzfy4zu7ioithky276aho5ehb6w3nu6q', + postUri: + 'at://did:plc:z72i7hdynmk6r22z27h6tvur/app.bsky.feed.post/3jzinucnmbi2c', + postAuthorDid: 'did:plc:z72i7hdynmk6r22z27h6tvur', + originatorDid: 'did:plc:z72i7hdynmk6r22z27h6tvur', + replyParent: null, + replyRoot: null, + sortAt: '2023-07-01T23:04:27.853Z', + }, + ], + } + // const { cursor, limit = 50 } = params + // const accountService = ctx.services.account(ctx.db) + // const feedService = ctx.services.appView.feed(ctx.db) + // const graphService = ctx.services.appView.graph(ctx.db) - const { ref } = ctx.db.db.dynamic + // const { ref } = ctx.db.db.dynamic - const keyset = new FeedKeyset(ref('post.indexedAt'), ref('post.cid')) - const sortFrom = keyset.unpack(cursor)?.primary + // const keyset = new FeedKeyset(ref('post.indexedAt'), ref('post.cid')) + // const sortFrom = keyset.unpack(cursor)?.primary - let postsQb = feedService - .selectPostQb() - .innerJoin('post_agg', 'post_agg.uri', 'post.uri') - .where('post_agg.likeCount', '>=', 5) - .whereExists((qb) => - qb - .selectFrom('follow') - .where('follow.creator', '=', requester) - .whereRef('follow.subjectDid', '=', 'post.creator'), - ) - .where((qb) => - accountService.whereNotMuted(qb, requester, [ref('post.creator')]), - ) - .whereNotExists(graphService.blockQb(requester, [ref('post.creator')])) - .where('post.indexedAt', '>', getFeedDateThreshold(sortFrom)) + // let postsQb = feedService + // .selectPostQb() + // // .innerJoin('post_agg', 'post_agg.uri', 'post.uri') + // // .where('post_agg.likeCount', '>=', 6) + // .whereExists((qb) => + // qb + // .selectFrom('follow') + // .where('follow.creator', '=', requester) + // .whereRef('follow.subjectDid', '=', 'post.creator'), + // ) + // .where((qb) => + // accountService.whereNotMuted(qb, requester, [ref('post.creator')]), + // ) + // .whereNotExists(graphService.blockQb(requester, [ref('post.creator')])) + // .where('post.indexedAt', '>', getFeedDateThreshold(sortFrom)) - postsQb = paginate(postsQb, { limit, cursor, keyset, tryIndex: true }) + // postsQb = paginate(postsQb, { limit, cursor, keyset, tryIndex: true }) - const feedItems = await postsQb.execute() - return { - feedItems, - cursor: keyset.packFromResult(feedItems), - } + // const feedItems = await postsQb.execute() + // return { + // feedItems, + // cursor: keyset.packFromResult(feedItems), + // } } export default handler diff --git a/packages/pds/src/index.ts b/packages/pds/src/index.ts index f77b654a069..3449425f63b 100644 --- a/packages/pds/src/index.ts +++ b/packages/pds/src/index.ts @@ -41,6 +41,7 @@ import { BackgroundQueue } from './event-stream/background-queue' import DidSqlCache from './did-cache' import { MountedAlgos } from './feed-gen/types' import { Crawlers } from './crawlers' +import { LabelCache } from './label-cache' export type { ServerConfigValues } from './config' export { ServerConfig } from './config' @@ -176,6 +177,8 @@ export class PDS { }) } + const labelCache = new LabelCache(db) + const services = createServices({ repoSigningKey, messageDispatcher, @@ -183,6 +186,7 @@ export class PDS { imgUriBuilder, imgInvalidator, labeler, + labelCache, backgroundQueue, crawlers, }) @@ -200,6 +204,7 @@ export class PDS { sequencer, sequencerLeader, labeler, + labelCache, services, mailer, imgUriBuilder, @@ -266,6 +271,7 @@ export class PDS { this.ctx.sequencerLeader.run() await this.ctx.sequencer.start() await this.ctx.db.startListeningToChannels() + this.ctx.labelCache.start() const server = this.app.listen(this.ctx.cfg.port) this.server = server this.server.keepAliveTimeout = 90000 @@ -275,6 +281,7 @@ export class PDS { } async destroy(): Promise { + this.ctx.labelCache.stop() await this.ctx.sequencerLeader.destroy() await this.terminator?.terminate() await this.ctx.backgroundQueue.destroy() diff --git a/packages/pds/src/label-cache.ts b/packages/pds/src/label-cache.ts new file mode 100644 index 00000000000..e4f23daa599 --- /dev/null +++ b/packages/pds/src/label-cache.ts @@ -0,0 +1,90 @@ +import { wait } from '@atproto/common' +import Database from './db' +import { Label } from './db/tables/label' +import { labelerLogger as log } from './logger' + +export class LabelCache { + bySubject: Record = {} + latestLabel = '' + refreshes = 0 + + destroyed = false + + constructor(public db: Database) {} + + start() { + this.poll() + } + + async fullRefresh() { + const allLabels = await this.db.db.selectFrom('label').selectAll().execute() + this.wipeCache() + this.processLabels(allLabels) + } + + async partialRefresh() { + const labels = await this.db.db + .selectFrom('label') + .selectAll() + .where('cts', '>', this.latestLabel) + .execute() + this.processLabels(labels) + } + + async poll() { + try { + if (this.destroyed) return + if (this.refreshes >= 120) { + await this.fullRefresh() + this.refreshes = 0 + } else { + await this.partialRefresh() + this.refreshes++ + } + } catch (err) { + log.error( + { err, latestLabel: this.latestLabel, refreshes: this.refreshes }, + 'label cache failed to refresh', + ) + } + await wait(500) + this.poll() + } + + processLabels(labels: Label[]) { + for (const label of labels) { + if (label.cts > this.latestLabel) { + this.latestLabel = label.cts + } + this.bySubject[label.uri] ??= [] + this.bySubject[label.uri].push(label) + } + } + + wipeCache() { + this.bySubject = {} + } + + stop() { + this.destroyed = true + } + + forSubject(subject: string, includeNeg = false): Label[] { + const labels = this.bySubject[subject] ?? [] + return includeNeg ? labels : labels.filter((l) => l.neg === 0) + } + + forSubjects(subjects: string[], includeNeg?: boolean): Label[] { + let labels: Label[] = [] + const alreadyAdded = new Set() + for (const subject of subjects) { + if (alreadyAdded.has(subject)) { + continue + } + const subLabels = this.forSubject(subject, includeNeg) + labels = [...labels, ...subLabels] + alreadyAdded.add(subject) + } + return labels + } +} diff --git a/packages/pds/src/sequencer/outbox.ts b/packages/pds/src/sequencer/outbox.ts index 335cd33cc3c..f0f62421a3d 100644 --- a/packages/pds/src/sequencer/outbox.ts +++ b/packages/pds/src/sequencer/outbox.ts @@ -108,14 +108,14 @@ export class Outbox { const evts = await this.sequencer.requestSeqRange({ earliestTime: backfillTime, earliestSeq: this.lastSeen > -1 ? this.lastSeen : backfillCursor, - limit: 10, + limit: 100, }) for (const evt of evts) { yield evt } // if we're within 50 of the sequencer, we call it good & switch to cutover const seqCursor = this.sequencer.lastSeen ?? -1 - if (seqCursor - this.lastSeen < 10) break + if (seqCursor - this.lastSeen < 100) break if (evts.length < 1) break } } diff --git a/packages/pds/src/sequencer/sequencer-leader.ts b/packages/pds/src/sequencer/sequencer-leader.ts index c951fda0d25..32964243f2e 100644 --- a/packages/pds/src/sequencer/sequencer-leader.ts +++ b/packages/pds/src/sequencer/sequencer-leader.ts @@ -1,5 +1,5 @@ import { DisconnectError } from '@atproto/xrpc-server' -import { jitter, wait } from '@atproto/common' +import { chunkArray, jitter, wait } from '@atproto/common' import { Leader } from '../db/leader' import { seqLogger as log } from '../logger' import Database from '../db' @@ -112,12 +112,20 @@ export class SequencerLeader { async sequenceOutgoing() { const unsequenced = await this.getUnsequenced() - for (const row of unsequenced) { - await this.db.db - .updateTable('repo_seq') - .set({ seq: this.nextSeqVal() }) - .where('id', '=', row.id) - .execute() + const chunks = chunkArray(unsequenced, 2000) + for (const chunk of chunks) { + await this.db.transaction(async (dbTxn) => { + await Promise.all( + chunk.map(async (row) => { + await dbTxn.db + .updateTable('repo_seq') + .set({ seq: this.nextSeqVal() }) + .where('id', '=', row.id) + .execute() + await this.db.notify('outgoing_repo_seq') + }), + ) + }) await this.db.notify('outgoing_repo_seq') } } diff --git a/packages/pds/src/sequencer/sequencer.ts b/packages/pds/src/sequencer/sequencer.ts index 4ea3b4ac836..61bcfa0efa7 100644 --- a/packages/pds/src/sequencer/sequencer.ts +++ b/packages/pds/src/sequencer/sequencer.ts @@ -126,7 +126,7 @@ export class Sequencer extends (EventEmitter as new () => SequencerEmitter) { try { const evts = await this.requestSeqRange({ earliestSeq: this.lastSeen, - limit: 10, + limit: 50, }) if (evts.length > 0) { this.emit('events', evts) diff --git a/packages/pds/src/services/index.ts b/packages/pds/src/services/index.ts index dd23198a8fa..f89fd917082 100644 --- a/packages/pds/src/services/index.ts +++ b/packages/pds/src/services/index.ts @@ -17,6 +17,7 @@ import { Labeler } from '../labeler' import { LabelService } from '../app-view/services/label' import { BackgroundQueue } from '../event-stream/background-queue' import { Crawlers } from '../crawlers' +import { LabelCache } from '../label-cache' export function createServices(resources: { repoSigningKey: crypto.Keypair @@ -25,6 +26,7 @@ export function createServices(resources: { imgUriBuilder: ImageUriBuilder imgInvalidator: ImageInvalidator labeler: Labeler + labelCache: LabelCache backgroundQueue: BackgroundQueue crawlers: Crawlers }): Services { @@ -35,6 +37,7 @@ export function createServices(resources: { imgUriBuilder, imgInvalidator, labeler, + labelCache, backgroundQueue, crawlers, } = resources @@ -57,11 +60,11 @@ export function createServices(resources: { imgInvalidator, ), appView: { - actor: ActorService.creator(imgUriBuilder), + actor: ActorService.creator(imgUriBuilder, labelCache), graph: GraphService.creator(imgUriBuilder), - feed: FeedService.creator(imgUriBuilder), + feed: FeedService.creator(imgUriBuilder, labelCache), indexing: IndexingService.creator(backgroundQueue), - label: LabelService.creator(), + label: LabelService.creator(labelCache), }, } } diff --git a/packages/pds/src/services/repo/index.ts b/packages/pds/src/services/repo/index.ts index c41e01b4c18..7dc1dac4252 100644 --- a/packages/pds/src/services/repo/index.ts +++ b/packages/pds/src/services/repo/index.ts @@ -132,10 +132,12 @@ export class RepoService { toWrite: { did: string; writes: PreparedWrite[]; swapCommitCid?: CID }, times: number, timeout = 100, + prevStorage?: SqlRepoStorage, ) { this.db.assertNotTransaction() const { did, writes, swapCommitCid } = toWrite - const storage = new SqlRepoStorage(this.db, did) + // we may have some useful cached blocks in the storage, so re-use the previous instance + const storage = prevStorage ?? new SqlRepoStorage(this.db, did) const commit = await this.formatCommit(storage, did, writes, swapCommitCid) try { await this.serviceTx(async (srvcTx) => @@ -147,7 +149,7 @@ export class RepoService { throw err } await wait(timeout) - return this.processWrites(toWrite, times - 1, timeout) + return this.processWrites(toWrite, times - 1, timeout, storage) } else { throw err } @@ -169,6 +171,8 @@ export class RepoService { if (swapCommit && !currRoot.equals(swapCommit)) { throw new BadCommitSwapError(currRoot) } + // cache last commit since there's likely overlap + await storage.cacheCommit(currRoot) const recordTxn = this.services.record(this.db) for (const write of writes) { const { action, uri, swapCid } = write diff --git a/packages/pds/src/sql-repo-storage.ts b/packages/pds/src/sql-repo-storage.ts index 3002ebe6ecf..0b9928c0371 100644 --- a/packages/pds/src/sql-repo-storage.ts +++ b/packages/pds/src/sql-repo-storage.ts @@ -42,6 +42,24 @@ export class SqlRepoStorage extends RepoStorage { return CID.parse(res.root) } + // proactively cache all blocks from a particular commit (to prevent multiple roundtrips) + async cacheCommit(cid: CID): Promise { + const res = await this.db.db + .selectFrom('repo_commit_block') + .innerJoin('ipld_block', (join) => + join + .onRef('ipld_block.cid', '=', 'repo_commit_block.block') + .onRef('ipld_block.creator', '=', 'repo_commit_block.creator'), + ) + .where('repo_commit_block.creator', '=', this.did) + .where('repo_commit_block.commit', '=', cid.toString()) + .select(['ipld_block.cid', 'ipld_block.content']) + .execute() + for (const row of res) { + this.cache.set(CID.parse(row.cid), row.content) + } + } + async getBytes(cid: CID): Promise { const cached = this.cache.get(cid) if (cached) return cached diff --git a/packages/pds/tests/_util.ts b/packages/pds/tests/_util.ts index 225b3a85a1d..115d62a4a9f 100644 --- a/packages/pds/tests/_util.ts +++ b/packages/pds/tests/_util.ts @@ -25,6 +25,7 @@ export type TestServerInfo = { url: string ctx: AppContext close: CloseFn + processAll: () => Promise } export type TestServerOpts = { @@ -154,6 +155,9 @@ export const runTestServer = async ( const pdsServer = await pds.start() const pdsPort = (pdsServer.address() as AddressInfo).port + // we refresh label cache by hand in `processAll` instead of on a timer + pds.ctx.labelCache.stop() + return { url: `http://localhost:${pdsPort}`, ctx: pds.ctx, @@ -161,6 +165,10 @@ export const runTestServer = async ( await pds.destroy() await plcServer.destroy() }, + processAll: async () => { + await pds.ctx.backgroundQueue.processAll() + await pds.ctx.labelCache.fullRefresh() + }, } } diff --git a/packages/pds/tests/account-deletion.test.ts b/packages/pds/tests/account-deletion.test.ts index 00004d78d7e..6720b52aee4 100644 --- a/packages/pds/tests/account-deletion.test.ts +++ b/packages/pds/tests/account-deletion.test.ts @@ -146,7 +146,7 @@ describe('account deletion', () => { did: carol.did, password: carol.password, }) - await server.ctx.backgroundQueue.processAll() // Finish background hard-deletions + await server.processAll() // Finish background hard-deletions }) it('no longer lets the user log in', async () => { diff --git a/packages/pds/tests/algos/hot-classic.test.ts b/packages/pds/tests/algos/hot-classic.test.ts index 180c4c90b92..1d5804f689a 100644 --- a/packages/pds/tests/algos/hot-classic.test.ts +++ b/packages/pds/tests/algos/hot-classic.test.ts @@ -35,7 +35,7 @@ describe('algo hot-classic', () => { alice = sc.dids.alice bob = sc.dids.bob - await server.ctx.backgroundQueue.processAll() + await server.processAll() }) afterAll(async () => { @@ -63,7 +63,7 @@ describe('algo hot-classic', () => { await sc.like(sc.dids[name], two.ref) await sc.like(sc.dids[name], three.ref) } - await server.ctx.backgroundQueue.processAll() + await server.processAll() const res = await agent.api.app.bsky.feed.getFeed( { feed: feedUri }, diff --git a/packages/pds/tests/algos/whats-hot.test.ts b/packages/pds/tests/algos/whats-hot.test.ts index d26c73dcf62..08d9f1a1b82 100644 --- a/packages/pds/tests/algos/whats-hot.test.ts +++ b/packages/pds/tests/algos/whats-hot.test.ts @@ -38,7 +38,7 @@ describe('algo whats-hot', () => { alice = sc.dids.alice bob = sc.dids.bob carol = sc.dids.carol - await server.ctx.backgroundQueue.processAll() + await server.processAll() }) afterAll(async () => { @@ -76,7 +76,7 @@ describe('algo whats-hot', () => { await sc.like(sc.dids[name], five.ref) } } - await server.ctx.backgroundQueue.processAll() + await server.processAll() // move the 3rd post 5 hours into the past to check gravity await server.ctx.db.db diff --git a/packages/pds/tests/algos/with-friends.test.ts b/packages/pds/tests/algos/with-friends.test.ts index 510274242cd..7e6e79357d9 100644 --- a/packages/pds/tests/algos/with-friends.test.ts +++ b/packages/pds/tests/algos/with-friends.test.ts @@ -40,7 +40,7 @@ describe.skip('algo with friends', () => { carol = sc.dids.carol dan = sc.dids.dan - await server.ctx.backgroundQueue.processAll() + await server.processAll() }) afterAll(async () => { diff --git a/packages/pds/tests/blob-deletes.test.ts b/packages/pds/tests/blob-deletes.test.ts index bf902b57169..aa8122423ba 100644 --- a/packages/pds/tests/blob-deletes.test.ts +++ b/packages/pds/tests/blob-deletes.test.ts @@ -58,7 +58,7 @@ describe('blob deletes', () => { ) const post = await sc.post(alice, 'test', undefined, [img]) await sc.deletePost(alice, post.ref.uri) - await server.ctx.backgroundQueue.processAll() + await server.processAll() const dbBlobs = await getDbBlobsForDid(alice) expect(dbBlobs.length).toBe(0) @@ -80,7 +80,7 @@ describe('blob deletes', () => { ) await updateProfile(sc, alice, img.image, img.image) await updateProfile(sc, alice, img2.image, img2.image) - await server.ctx.backgroundQueue.processAll() + await server.processAll() const dbBlobs = await getDbBlobsForDid(alice) expect(dbBlobs.length).toBe(1) @@ -109,7 +109,7 @@ describe('blob deletes', () => { ) await updateProfile(sc, alice, img.image, img.image) await updateProfile(sc, alice, img.image, img2.image) - await server.ctx.backgroundQueue.processAll() + await server.processAll() const dbBlobs = await getDbBlobsForDid(alice) expect(dbBlobs.length).toBe(2) @@ -160,7 +160,7 @@ describe('blob deletes', () => { }, { encoding: 'application/json', headers: sc.getHeaders(alice) }, ) - await server.ctx.backgroundQueue.processAll() + await server.processAll() const dbBlobs = await getDbBlobsForDid(alice) expect(dbBlobs.length).toBe(1) diff --git a/packages/pds/tests/event-stream/sync.test.ts b/packages/pds/tests/event-stream/sync.test.ts index cf42f86223c..a596b7240ce 100644 --- a/packages/pds/tests/event-stream/sync.test.ts +++ b/packages/pds/tests/event-stream/sync.test.ts @@ -68,6 +68,7 @@ describe('sync', () => { services.repo(dbTxn).indexWrites(writes, now), ) } + await server.processAll() // Check indexed timeline const aliceTL = await agent.api.app.bsky.feed.getTimeline( {}, diff --git a/packages/pds/tests/feed-generation.test.ts b/packages/pds/tests/feed-generation.test.ts index e5c9ab8e9c2..caa1f38423c 100644 --- a/packages/pds/tests/feed-generation.test.ts +++ b/packages/pds/tests/feed-generation.test.ts @@ -38,7 +38,7 @@ describe('feed generation', () => { agent = network.pds.getClient() sc = new SeedClient(agent) await basicSeed(sc) - await network.pds.ctx.backgroundQueue.processAll() + await network.processAll() alice = sc.dids.alice const allUri = AtUri.make(alice, 'app.bsky.feed.generator', 'all') const feedUriBadPagination = AtUri.make( diff --git a/packages/pds/tests/indexing.test.ts b/packages/pds/tests/indexing.test.ts index 7402d7e6aa9..4aede170617 100644 --- a/packages/pds/tests/indexing.test.ts +++ b/packages/pds/tests/indexing.test.ts @@ -82,7 +82,7 @@ describe('indexing', () => { await services .repo(db) .processWrites({ did: sc.dids.alice, writes: [createRecord] }, 1) - await server.ctx.backgroundQueue.processAll() + await server.processAll() const getAfterCreate = await agent.api.app.bsky.feed.getPostThread( { uri: uri.toString() }, @@ -95,7 +95,7 @@ describe('indexing', () => { await services .repo(db) .processWrites({ did: sc.dids.alice, writes: [updateRecord] }, 1) - await server.ctx.backgroundQueue.processAll() + await server.processAll() const getAfterUpdate = await agent.api.app.bsky.feed.getPostThread( { uri: uri.toString() }, @@ -108,7 +108,7 @@ describe('indexing', () => { await services .repo(db) .processWrites({ did: sc.dids.alice, writes: [deleteRecord] }, 1) - await server.ctx.backgroundQueue.processAll() + await server.processAll() const getAfterDelete = agent.api.app.bsky.feed.getPostThread( { uri: uri.toString() }, @@ -157,7 +157,7 @@ describe('indexing', () => { await services .repo(db) .processWrites({ did: sc.dids.dan, writes: [createRecord] }, 1) - await server.ctx.backgroundQueue.processAll() + await server.processAll() const getAfterCreate = await agent.api.app.bsky.actor.getProfile( { actor: sc.dids.dan }, @@ -170,7 +170,7 @@ describe('indexing', () => { await services .repo(db) .processWrites({ did: sc.dids.dan, writes: [updateRecord] }, 1) - await server.ctx.backgroundQueue.processAll() + await server.processAll() const getAfterUpdate = await agent.api.app.bsky.actor.getProfile( { actor: sc.dids.dan }, @@ -183,7 +183,7 @@ describe('indexing', () => { await services .repo(db) .processWrites({ did: sc.dids.dan, writes: [deleteRecord] }, 1) - await server.ctx.backgroundQueue.processAll() + await server.processAll() const getAfterDelete = await agent.api.app.bsky.actor.getProfile( { actor: sc.dids.dan }, diff --git a/packages/pds/tests/labeler/labeler.test.ts b/packages/pds/tests/labeler/labeler.test.ts index ea943dcf46d..2525732357a 100644 --- a/packages/pds/tests/labeler/labeler.test.ts +++ b/packages/pds/tests/labeler/labeler.test.ts @@ -1,6 +1,6 @@ import { AtUri, BlobRef } from '@atproto/api' import stream from 'stream' -import { runTestServer, CloseFn } from '../_util' +import { runTestServer, CloseFn, TestServerInfo } from '../_util' import { Labeler } from '../../src/labeler' import { AppContext, Database } from '../../src' import { BlobStore, cidForRecord } from '@atproto/repo' @@ -11,6 +11,7 @@ import { LabelService } from '../../src/app-view/services/label' import { BackgroundQueue } from '../../src/event-stream/background-queue' describe('labeler', () => { + let server: TestServerInfo let close: CloseFn let labeler: Labeler let labelSrvc: LabelService @@ -21,7 +22,7 @@ describe('labeler', () => { let goodBlob: BlobRef beforeAll(async () => { - const server = await runTestServer({ + server = await runTestServer({ dbPostgresSchema: 'views_author_feed', }) close = server.close @@ -63,6 +64,7 @@ describe('labeler', () => { const uri = postUri() labeler.processRecord(uri, post) await labeler.processAll() + await server.processAll() const labels = await labelSrvc.getLabels(uri.toString()) expect(labels.length).toBe(1) expect(labels[0]).toMatchObject({ @@ -100,6 +102,7 @@ describe('labeler', () => { const uri = postUri() labeler.processRecord(uri, post) await labeler.processAll() + await server.processAll() const dbLabels = await labelSrvc.getLabels(uri.toString()) const labels = dbLabels.map((row) => row.val).sort() expect(labels).toEqual( @@ -119,6 +122,7 @@ describe('labeler', () => { cts: new Date().toISOString(), }) .execute() + await server.processAll() const labels = await labelSrvc.getLabelsForProfile('did:example:alice') // 4 from earlier & then just added one diff --git a/packages/pds/tests/migrations/blob-creator.test.ts b/packages/pds/tests/migrations/blob-creator.test.ts index 8d6843a1136..defe3bde6f4 100644 --- a/packages/pds/tests/migrations/blob-creator.test.ts +++ b/packages/pds/tests/migrations/blob-creator.test.ts @@ -4,7 +4,7 @@ import { cidForCbor, TID } from '@atproto/common' import { Kysely } from 'kysely' import { AtUri } from '@atproto/uri' -describe('blob creator migration', () => { +describe.skip('blob creator migration', () => { let db: Database let rawDb: Kysely diff --git a/packages/pds/tests/migrations/indexed-at-on-record.test.ts b/packages/pds/tests/migrations/indexed-at-on-record.test.ts index d2a3f8f4f8f..02147816203 100644 --- a/packages/pds/tests/migrations/indexed-at-on-record.test.ts +++ b/packages/pds/tests/migrations/indexed-at-on-record.test.ts @@ -4,7 +4,7 @@ import { dataToCborBlock, TID } from '@atproto/common' import { AtUri } from '@atproto/uri' import { Kysely } from 'kysely' -describe('indexedAt on record migration', () => { +describe.skip('indexedAt on record migration', () => { let db: Database let rawDb: Kysely diff --git a/packages/pds/tests/migrations/post-hierarchy.test.ts b/packages/pds/tests/migrations/post-hierarchy.test.ts index 145be7bb16c..3e4a0a4ff3d 100644 --- a/packages/pds/tests/migrations/post-hierarchy.test.ts +++ b/packages/pds/tests/migrations/post-hierarchy.test.ts @@ -5,7 +5,7 @@ import usersSeed from '../seeds/users' import threadSeed, { walk, item, Item } from '../seeds/thread' import { CloseFn, runTestServer } from '../_util' -describe('post hierarchy migration', () => { +describe.skip('post hierarchy migration', () => { let db: Database let close: CloseFn let sc: SeedClient diff --git a/packages/pds/tests/migrations/repo-sync-data-pt2.test.ts b/packages/pds/tests/migrations/repo-sync-data-pt2.test.ts index ad046836ed6..91971fb7043 100644 --- a/packages/pds/tests/migrations/repo-sync-data-pt2.test.ts +++ b/packages/pds/tests/migrations/repo-sync-data-pt2.test.ts @@ -5,7 +5,7 @@ import { CloseFn, runTestServer } from '../_util' import { SeedClient } from '../seeds/client' import basicSeed from '../seeds/basic' -describe('repo sync data migration', () => { +describe.skip('repo sync data migration', () => { let db: Database let rawDb: Kysely let close: CloseFn diff --git a/packages/pds/tests/migrations/repo-sync-data.test.ts b/packages/pds/tests/migrations/repo-sync-data.test.ts index 4b56aede9db..2fd17c199eb 100644 --- a/packages/pds/tests/migrations/repo-sync-data.test.ts +++ b/packages/pds/tests/migrations/repo-sync-data.test.ts @@ -5,7 +5,7 @@ import { TID } from '@atproto/common' import { Kysely } from 'kysely' import { CID } from 'multiformats/cid' -describe('repo sync data migration', () => { +describe.skip('repo sync data migration', () => { let db: Database let rawDb: Kysely let memoryStore: MemoryBlockstore diff --git a/packages/pds/tests/migrations/user-partitioned-cids.test.ts b/packages/pds/tests/migrations/user-partitioned-cids.test.ts index 048ccd9dacf..e6eff220445 100644 --- a/packages/pds/tests/migrations/user-partitioned-cids.test.ts +++ b/packages/pds/tests/migrations/user-partitioned-cids.test.ts @@ -5,7 +5,7 @@ import { Kysely } from 'kysely' import { Block } from 'multiformats/block' import * as uint8arrays from 'uint8arrays' -describe('user partitioned cids migration', () => { +describe.skip('user partitioned cids migration', () => { let db: Database let rawDb: Kysely diff --git a/packages/pds/tests/migrations/user-table-did-pkey.test.ts b/packages/pds/tests/migrations/user-table-did-pkey.test.ts index bda821a1539..881907e71b8 100644 --- a/packages/pds/tests/migrations/user-table-did-pkey.test.ts +++ b/packages/pds/tests/migrations/user-table-did-pkey.test.ts @@ -2,7 +2,7 @@ import { Database } from '../../src' import { randomStr } from '@atproto/crypto' import { Kysely } from 'kysely' -describe('user table did pkey migration', () => { +describe.skip('user table did pkey migration', () => { let db: Database let rawDb: Kysely diff --git a/packages/pds/tests/moderation.test.ts b/packages/pds/tests/moderation.test.ts index caf645b3c9d..19bcbee1ad3 100644 --- a/packages/pds/tests/moderation.test.ts +++ b/packages/pds/tests/moderation.test.ts @@ -292,7 +292,7 @@ describe('moderation', () => { password: 'password', token: deletionToken, }) - await server.ctx.backgroundQueue.processAll() + await server.processAll() // Take action on deleted content const { data: action } = await agent.api.com.atproto.admin.takeModerationAction( diff --git a/packages/pds/tests/proxied/feedgen.test.ts b/packages/pds/tests/proxied/feedgen.test.ts index 2914813750d..44a89e0b34c 100644 --- a/packages/pds/tests/proxied/feedgen.test.ts +++ b/packages/pds/tests/proxied/feedgen.test.ts @@ -36,7 +36,6 @@ describe('feedgen proxy view', () => { sc.getHeaders(sc.dids.alice), ) await network.processAll() - await network.bsky.ctx.backgroundQueue.processAll() // mock getFeedGenerator() for use by pds's getFeed since we don't have a proper feedGenDid or feed publisher FeedNS.prototype.getFeedGenerator = async function (params, opts) { if (params?.feed === feedUri.toString()) { diff --git a/packages/pds/tests/views/actor-search.test.ts b/packages/pds/tests/views/actor-search.test.ts index 5cb778db4a7..f778f1387c4 100644 --- a/packages/pds/tests/views/actor-search.test.ts +++ b/packages/pds/tests/views/actor-search.test.ts @@ -28,7 +28,7 @@ describe('pds user search views', () => { sc = new SeedClient(agent) await usersBulkSeed(sc) headers = sc.getHeaders(Object.values(sc.dids)[0]) - await server.ctx.backgroundQueue.processAll() + await server.processAll() }) afterAll(async () => { diff --git a/packages/pds/tests/views/author-feed.test.ts b/packages/pds/tests/views/author-feed.test.ts index 68cdd0bc48b..64246f0fcf2 100644 --- a/packages/pds/tests/views/author-feed.test.ts +++ b/packages/pds/tests/views/author-feed.test.ts @@ -33,7 +33,7 @@ describe('pds author feed views', () => { bob = sc.dids.bob carol = sc.dids.carol dan = sc.dids.dan - await server.ctx.backgroundQueue.processAll() + await server.processAll() }) afterAll(async () => { diff --git a/packages/pds/tests/views/blocks.test.ts b/packages/pds/tests/views/blocks.test.ts index 4d72e639b1d..a3815a6a668 100644 --- a/packages/pds/tests/views/blocks.test.ts +++ b/packages/pds/tests/views/blocks.test.ts @@ -43,7 +43,7 @@ describe('pds views with blocking', () => { sc.posts[dan][0].ref, 'alice replies to dan', ) - await server.ctx.backgroundQueue.processAll() + await server.processAll() }) afterAll(async () => { diff --git a/packages/pds/tests/views/follows.test.ts b/packages/pds/tests/views/follows.test.ts index 982aa2c0d29..606b85f9d48 100644 --- a/packages/pds/tests/views/follows.test.ts +++ b/packages/pds/tests/views/follows.test.ts @@ -27,7 +27,7 @@ describe('pds follow views', () => { sc = new SeedClient(agent) await followsSeed(sc) alice = sc.dids.alice - await server.ctx.backgroundQueue.processAll() + await server.processAll() }) afterAll(async () => { diff --git a/packages/pds/tests/views/likes.test.ts b/packages/pds/tests/views/likes.test.ts index 10bef798aba..d77251d8b25 100644 --- a/packages/pds/tests/views/likes.test.ts +++ b/packages/pds/tests/views/likes.test.ts @@ -28,7 +28,7 @@ describe('pds like views', () => { await likesSeed(sc) alice = sc.dids.alice bob = sc.dids.bob - await server.ctx.backgroundQueue.processAll() + await server.processAll() }) afterAll(async () => { diff --git a/packages/pds/tests/views/mute-lists.test.ts b/packages/pds/tests/views/mute-lists.test.ts index 0cf15770617..64692a81b24 100644 --- a/packages/pds/tests/views/mute-lists.test.ts +++ b/packages/pds/tests/views/mute-lists.test.ts @@ -30,7 +30,7 @@ describe('pds views with mutes from mute lists', () => { // add follows to ensure mutes work even w follows await sc.follow(carol, dan) await sc.follow(dan, carol) - await server.ctx.backgroundQueue.processAll() + await server.processAll() }) afterAll(async () => { diff --git a/packages/pds/tests/views/mutes.test.ts b/packages/pds/tests/views/mutes.test.ts index a3a17eff595..ac32d43ab6f 100644 --- a/packages/pds/tests/views/mutes.test.ts +++ b/packages/pds/tests/views/mutes.test.ts @@ -32,7 +32,7 @@ describe('mute views', () => { { headers: sc.getHeaders(silas), encoding: 'application/json' }, ) } - await server.ctx.backgroundQueue.processAll() + await server.processAll() }) afterAll(async () => { diff --git a/packages/pds/tests/views/notifications.test.ts b/packages/pds/tests/views/notifications.test.ts index 67b00331e3c..b5d9907140c 100644 --- a/packages/pds/tests/views/notifications.test.ts +++ b/packages/pds/tests/views/notifications.test.ts @@ -33,7 +33,7 @@ describe('pds notification views', () => { sc = new SeedClient(agent) await basicSeed(sc) alice = sc.dids.alice - await server.ctx.backgroundQueue.processAll() + await server.processAll() }) afterAll(async () => { @@ -75,7 +75,7 @@ describe('pds notification views', () => { sc.replies[alice][0].ref, 'indeed', ) - await server.ctx.backgroundQueue.processAll() + await server.processAll() const notifCountAlice = await agent.api.app.bsky.notification.getUnreadCount( diff --git a/packages/pds/tests/views/popular.test.ts b/packages/pds/tests/views/popular.test.ts index 1456f4cbc33..ca21ee31f4f 100644 --- a/packages/pds/tests/views/popular.test.ts +++ b/packages/pds/tests/views/popular.test.ts @@ -54,7 +54,7 @@ describe('popular views', () => { alice = sc.dids.alice bob = sc.dids.bob - await server.ctx.backgroundQueue.processAll() + await server.processAll() }) afterAll(async () => { @@ -82,7 +82,7 @@ describe('popular views', () => { await sc.like(sc.dids[name], two.ref) await sc.like(sc.dids[name], three.ref) } - await server.ctx.backgroundQueue.processAll() + await server.processAll() const res = await agent.api.app.bsky.unspecced.getPopular( {}, diff --git a/packages/pds/tests/views/posts.test.ts b/packages/pds/tests/views/posts.test.ts index 55b2312cf12..7bf6e6919cf 100644 --- a/packages/pds/tests/views/posts.test.ts +++ b/packages/pds/tests/views/posts.test.ts @@ -15,7 +15,7 @@ describe('pds posts views', () => { agent = new AtpAgent({ service: server.url }) sc = new SeedClient(agent) await basicSeed(sc) - await server.ctx.labeler.processAll() + await server.processAll() }) afterAll(async () => { diff --git a/packages/pds/tests/views/profile.test.ts b/packages/pds/tests/views/profile.test.ts index 1f68946811d..b3f5520a8b9 100644 --- a/packages/pds/tests/views/profile.test.ts +++ b/packages/pds/tests/views/profile.test.ts @@ -27,7 +27,7 @@ describe('pds profile views', () => { alice = sc.dids.alice bob = sc.dids.bob dan = sc.dids.dan - await server.ctx.backgroundQueue.processAll() + await server.processAll() }) afterAll(async () => { diff --git a/packages/pds/tests/views/reposts.test.ts b/packages/pds/tests/views/reposts.test.ts index 5e4eb6e0b37..1c61215e3e5 100644 --- a/packages/pds/tests/views/reposts.test.ts +++ b/packages/pds/tests/views/reposts.test.ts @@ -22,7 +22,7 @@ describe('pds repost views', () => { await repostsSeed(sc) alice = sc.dids.alice bob = sc.dids.bob - await server.ctx.backgroundQueue.processAll() + await server.processAll() }) afterAll(async () => { diff --git a/packages/pds/tests/views/suggestions.test.ts b/packages/pds/tests/views/suggestions.test.ts index 1c075fe17a8..4bec6dc3dd2 100644 --- a/packages/pds/tests/views/suggestions.test.ts +++ b/packages/pds/tests/views/suggestions.test.ts @@ -16,7 +16,7 @@ describe('pds user search views', () => { agent = new AtpAgent({ service: server.url }) sc = new SeedClient(agent) await basicSeed(sc) - await server.ctx.backgroundQueue.processAll() + await server.processAll() const suggestions = [ { did: sc.dids.bob, order: 1 }, diff --git a/packages/pds/tests/views/thread.test.ts b/packages/pds/tests/views/thread.test.ts index 0f4fa68b35e..8838ea20e44 100644 --- a/packages/pds/tests/views/thread.test.ts +++ b/packages/pds/tests/views/thread.test.ts @@ -33,6 +33,7 @@ describe('pds thread views', () => { agent = new AtpAgent({ service: server.url }) sc = new SeedClient(agent) await basicSeed(sc) + await server.processAll() alice = sc.dids.alice bob = sc.dids.bob carol = sc.dids.carol @@ -41,7 +42,7 @@ describe('pds thread views', () => { beforeAll(async () => { // Add a repost of a reply so that we can confirm viewer state in the thread await sc.repost(bob, sc.replies[alice][0].ref) - await server.ctx.backgroundQueue.processAll() + await server.processAll() }) afterAll(async () => { @@ -140,7 +141,7 @@ describe('pds thread views', () => { 'Reply reply', ) indexes.aliceReplyReply = sc.replies[alice].length - 1 - await server.ctx.backgroundQueue.processAll() + await server.processAll() const thread1 = await agent.api.app.bsky.feed.getPostThread( { uri: sc.posts[alice][indexes.aliceRoot].ref.uriStr }, @@ -149,7 +150,7 @@ describe('pds thread views', () => { expect(forSnapshot(thread1.data.thread)).toMatchSnapshot() await sc.deletePost(bob, sc.replies[bob][indexes.bobReply].ref.uri) - await server.ctx.backgroundQueue.processAll() + await server.processAll() const thread2 = await agent.api.app.bsky.feed.getPostThread( { uri: sc.posts[alice][indexes.aliceRoot].ref.uriStr }, diff --git a/packages/pds/tests/views/timeline.test.ts b/packages/pds/tests/views/timeline.test.ts index dc0d10a79cb..b55a96e691d 100644 --- a/packages/pds/tests/views/timeline.test.ts +++ b/packages/pds/tests/views/timeline.test.ts @@ -55,7 +55,7 @@ describe('timeline views', () => { labelPostB.cidStr, { create: ['kind'] }, ) - await server.ctx.backgroundQueue.processAll() + await server.processAll() }) afterAll(async () => {