From 89d28ea9b1fa5d223a526d99a491da31475a3cb9 Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Wed, 2 Aug 2023 12:08:25 -0500 Subject: [PATCH 01/20] feat: queue --- .../src/call-executors/queue-call-executor.ts | 10 + .../core-runtime/src/clients/queue-client.ts | 11 + .../core-runtime/src/handlers/worker.ts | 12 + .../queue-name-property-retriever.ts | 12 + packages/@eventual/core/src/index.ts | 1 + packages/@eventual/core/src/internal/calls.ts | 35 ++- .../@eventual/core/src/internal/properties.ts | 7 + .../@eventual/core/src/internal/resources.ts | 16 +- .../core/src/internal/service-spec.ts | 33 +++ packages/@eventual/core/src/queue/fifo.ts | 0 packages/@eventual/core/src/queue/index.ts | 2 + packages/@eventual/core/src/queue/queue.ts | 273 ++++++++++++++++++ 12 files changed, 404 insertions(+), 8 deletions(-) create mode 100644 packages/@eventual/core-runtime/src/call-executors/queue-call-executor.ts create mode 100644 packages/@eventual/core-runtime/src/clients/queue-client.ts create mode 100644 packages/@eventual/core-runtime/src/property-retrievers/queue-name-property-retriever.ts create mode 100644 packages/@eventual/core/src/queue/fifo.ts create mode 100644 packages/@eventual/core/src/queue/index.ts create mode 100644 packages/@eventual/core/src/queue/queue.ts diff --git a/packages/@eventual/core-runtime/src/call-executors/queue-call-executor.ts b/packages/@eventual/core-runtime/src/call-executors/queue-call-executor.ts new file mode 100644 index 000000000..d721f91a8 --- /dev/null +++ b/packages/@eventual/core-runtime/src/call-executors/queue-call-executor.ts @@ -0,0 +1,10 @@ +import { QueueCall } from "@eventual/core/internal"; +import type { CallExecutor } from "../call-executor.js"; +import { QueueClient } from "../clients/queue-client.js"; + +export class QueueCallExecutor implements CallExecutor { + constructor(private queueClient: QueueClient) {} + public execute(call: QueueCall): Promise { + return this.queueClient[call.operation](call.queueName, ...call.params); + } +} diff --git a/packages/@eventual/core-runtime/src/clients/queue-client.ts b/packages/@eventual/core-runtime/src/clients/queue-client.ts new file mode 100644 index 000000000..76e2a660a --- /dev/null +++ b/packages/@eventual/core-runtime/src/clients/queue-client.ts @@ -0,0 +1,11 @@ +import { Queue } from "@eventual/core"; +import { QueueMethod } from "@eventual/core/internal"; + +export type QueueClient = { + [K in keyof Pick]: ( + queueName: string, + ...args: Parameters + ) => ReturnType; +} & { + physicalName: (queueName: string) => string; +}; \ No newline at end of file diff --git a/packages/@eventual/core-runtime/src/handlers/worker.ts b/packages/@eventual/core-runtime/src/handlers/worker.ts index f255fc6fe..189f97912 100644 --- a/packages/@eventual/core-runtime/src/handlers/worker.ts +++ b/packages/@eventual/core-runtime/src/handlers/worker.ts @@ -8,9 +8,11 @@ import { import { AwaitTimerCallPassthroughExecutor } from "../call-executors/await-timer-executor.js"; import { BucketCallExecutor } from "../call-executors/bucket-call-executor.js"; import { EntityCallExecutor } from "../call-executors/entity-call-executor.js"; +import { QueueCallExecutor } from "../call-executors/queue-call-executor.js"; import { SearchCallExecutor } from "../call-executors/search-call-executor.js"; import { ServiceClientExecutor } from "../call-executors/service-client-executor.js"; import type { OpenSearchClient } from "../clients/open-search-client.js"; +import type { QueueClient } from "../clients/queue-client.js"; import { enterEventualCallHookScope } from "../eventual-hook.js"; import { AllPropertyRetriever, @@ -19,6 +21,7 @@ import { } from "../property-retriever.js"; import { BucketPhysicalNamePropertyRetriever } from "../property-retrievers/bucket-name-property-retriever.js"; import { OpenSearchClientPropertyRetriever } from "../property-retrievers/open-search-client-property-retriever.js"; +import { QueuePhysicalNamePropertyRetriever } from "../property-retrievers/queue-name-property-retriever.js"; import type { BucketStore } from "../stores/bucket-store.js"; import type { EntityStore } from "../stores/entity-store.js"; import type { LazyValue } from "../utils.js"; @@ -27,6 +30,7 @@ export interface WorkerIntrinsicDeps { bucketStore: BucketStore | undefined; entityStore: EntityStore | undefined; openSearchClient: OpenSearchClient | undefined; + queueClient: QueueClient | undefined; serviceClient: EventualServiceClient | undefined; serviceName: string | LazyValue; serviceSpec: ServiceSpec | undefined; @@ -77,6 +81,12 @@ export function createEventualWorker( const entityCallExecutor = props.entityStore ? new EntityCallExecutor(props.entityStore) : unsupportedExecutor; + const queueCallExecutor = props.queueClient + ? new QueueCallExecutor(props.queueClient) + : unsupportedExecutor; + const queuePhysicalNamePropertyRetriever = props.queueClient + ? new QueuePhysicalNamePropertyRetriever(props.queueClient) + : unsupportedProperty; return (...input: Input) => { const resolvedExecutorOverrides = props.executorOverrides @@ -119,6 +129,7 @@ export function createEventualWorker( ExpectSignalCall: unsupportedExecutor, GetExecutionCall: serviceClientExecutor, InvokeTransactionCall: serviceClientExecutor, + QueueCall: queueCallExecutor, // register signal handler does not work outside of a workflow SignalHandlerCall: unsupportedExecutor, SearchCall: openSearchExecutor, @@ -132,6 +143,7 @@ export function createEventualWorker( new AllPropertyRetriever({ BucketPhysicalName: bucketPhysicalNameRetriever, OpenSearchClient: openSearchClientPropertyRetriever, + QueuePhysicalName: queuePhysicalNamePropertyRetriever, ServiceClient: props.serviceClient ?? unsupportedProperty, ServiceName: props.serviceName, ServiceSpec: props.serviceSpec ?? unsupportedProperty, diff --git a/packages/@eventual/core-runtime/src/property-retrievers/queue-name-property-retriever.ts b/packages/@eventual/core-runtime/src/property-retrievers/queue-name-property-retriever.ts new file mode 100644 index 000000000..62bd873e0 --- /dev/null +++ b/packages/@eventual/core-runtime/src/property-retrievers/queue-name-property-retriever.ts @@ -0,0 +1,12 @@ +import type { QueuePhysicalName } from "@eventual/core/internal"; +import type { QueueClient } from "../clients/queue-client.js"; +import type { PropertyResolver } from "../property-retriever.js"; + +export class QueuePhysicalNamePropertyRetriever + implements PropertyResolver +{ + constructor(private queueClient: QueueClient) {} + public getProperty(property: QueuePhysicalName): string { + return this.queueClient.physicalName(property.queueName); + } +} diff --git a/packages/@eventual/core/src/index.ts b/packages/@eventual/core/src/index.ts index 5138332cd..87439afbe 100644 --- a/packages/@eventual/core/src/index.ts +++ b/packages/@eventual/core/src/index.ts @@ -11,6 +11,7 @@ export * from "./http-method.js"; export * from "./http/index.js"; export * from "./infer.js"; export * from "./logging.js"; +export * from "./queue/index.js"; export * from "./schedule.js"; export * from "./search/index.js"; export * from "./secret.js"; diff --git a/packages/@eventual/core/src/internal/calls.ts b/packages/@eventual/core/src/internal/calls.ts index 56d5de40e..dd7e35380 100644 --- a/packages/@eventual/core/src/internal/calls.ts +++ b/packages/@eventual/core/src/internal/calls.ts @@ -7,6 +7,7 @@ import type { } from "../entity/entity.js"; import type { EventEnvelope } from "../event.js"; import type { Execution, ExecutionHandle } from "../execution.js"; +import type { Queue } from "../queue/queue.js"; import type { DurationSchedule, Schedule } from "../schedule.js"; import type { SearchIndex } from "../search/search-index.js"; import type { Task } from "../task.js"; @@ -28,7 +29,8 @@ export type Call = | SendSignalCall | StartWorkflowCall | TaskCall - | TaskRequestCall; + | TaskRequestCall + | QueueCall; export enum CallKind { AwaitTimerCall = 1, @@ -46,6 +48,7 @@ export enum CallKind { TaskRequestCall = 12, SearchCall = 11, StartWorkflowCall = 13, + QueueCall = 15, } export const CallSymbol = /* @__PURE__ */ Symbol.for("eventual:EventualCall"); @@ -221,6 +224,36 @@ export function isBucketCallOperation( return operation.operation.operation === op; } +export function isQueueCall(a: any): a is QueueCall { + return isCallOfKind(CallKind.QueueCall, a); +} + +export type QueueMethod = Exclude< + { + [k in keyof Queue]: Queue[k] extends Function ? k : never; + }[keyof Queue], + "forEach" | "forEachBatch" | undefined +>; + +export type QueueCall = CallBase< + CallKind.QueueCall, + ReturnType +> & + QueueOperation; + +export type QueueOperation = { + operation: Op; + queueName: string; + params: Parameters; +}; + +export function isQueueCallType( + op: Op, + operation: QueueCall +): operation is QueueCall { + return operation.operation === op; +} + export function isExpectSignalCall(a: any): a is ExpectSignalCall { return isCallOfKind(CallKind.ExpectSignalCall, a); } diff --git a/packages/@eventual/core/src/internal/properties.ts b/packages/@eventual/core/src/internal/properties.ts index 14f2a0bb2..be6a3c015 100644 --- a/packages/@eventual/core/src/internal/properties.ts +++ b/packages/@eventual/core/src/internal/properties.ts @@ -5,6 +5,7 @@ import type { ServiceType } from "./service-type.js"; export enum PropertyKind { BucketPhysicalName, + QueuePhysicalName, OpenSearchClient, ServiceClient, ServiceName, @@ -21,6 +22,7 @@ export const PropertySymbol = /* @__PURE__ */ Symbol.for( export type Property = | BucketPhysicalName | OpenSearchClientProperty + | QueuePhysicalName | ServiceClientProperty | ServiceNameProperty | ServiceSpecProperty @@ -60,6 +62,11 @@ export interface BucketPhysicalName bucketName: string; } +export interface QueuePhysicalName + extends PropertyBase { + queueName: string; +} + export type OpenSearchClientProperty = PropertyBase< PropertyKind.OpenSearchClient, OpenSearchClient diff --git a/packages/@eventual/core/src/internal/resources.ts b/packages/@eventual/core/src/internal/resources.ts index 3a509b222..28bcb0887 100644 --- a/packages/@eventual/core/src/internal/resources.ts +++ b/packages/@eventual/core/src/internal/resources.ts @@ -2,6 +2,7 @@ import type { Bucket } from "../bucket.js"; import type { Entity } from "../entity/entity.js"; import type { Event } from "../event.js"; import type { AnyCommand } from "../http/command.js"; +import type { Queue } from "../queue/queue.js"; import type { SearchIndex } from "../search/search-index.js"; import type { Subscription } from "../subscription.js"; import type { Task } from "../task.js"; @@ -9,15 +10,16 @@ import type { Transaction } from "../transaction.js"; import type { Workflow } from "../workflow.js"; type Resource = - | Task - | Workflow - | Transaction - | Entity - | Bucket - | SearchIndex | AnyCommand + | Bucket + | Entity | Event - | Subscription; + | SearchIndex + | Subscription + | Task + | Transaction + | Queue + | Workflow; type ResourceKind = Resource["kind"]; diff --git a/packages/@eventual/core/src/internal/service-spec.ts b/packages/@eventual/core/src/internal/service-spec.ts index 63ff19bc8..9eaaaf57b 100644 --- a/packages/@eventual/core/src/internal/service-spec.ts +++ b/packages/@eventual/core/src/internal/service-spec.ts @@ -49,6 +49,7 @@ export interface ServiceSpec { search: { indices: IndexSpec[]; }; + queues: QueueSpec[]; } export interface FunctionSpec { @@ -288,3 +289,35 @@ export interface EntityIndexSpec { export interface TransactionSpec { name: Name; } + +/** + * TODO: Support filter criteria. + */ +export interface QueueHandlerOptions extends FunctionRuntimeProps { + /** + * Max batch size. Between 1 and 1000. + * + * @default: 100 + */ + batchSize?: number; + /** + * Amount of time to wait for the batch size before sending a batch. + * + * @default: 0 seconds. + */ + batchingWindow?: DurationSchedule; +} + +export interface QueueHandlerSpec { + name: Name; + queueName: string; + options?: EntityStreamOptions; + sourceLocation?: SourceLocation; +} + +export interface QueueSpec { + name: Name; + handlers: QueueHandlerSpec[]; + message?: openapi.SchemaObject; + attributes?: openapi.SchemaObject; +} diff --git a/packages/@eventual/core/src/queue/fifo.ts b/packages/@eventual/core/src/queue/fifo.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/@eventual/core/src/queue/index.ts b/packages/@eventual/core/src/queue/index.ts new file mode 100644 index 000000000..30ecf60a9 --- /dev/null +++ b/packages/@eventual/core/src/queue/index.ts @@ -0,0 +1,2 @@ +export * from "./queue.js"; +export * from "./fifo.js"; diff --git a/packages/@eventual/core/src/queue/queue.ts b/packages/@eventual/core/src/queue/queue.ts new file mode 100644 index 000000000..ec969b601 --- /dev/null +++ b/packages/@eventual/core/src/queue/queue.ts @@ -0,0 +1,273 @@ +import { z } from "zod"; +import { CallKind, createCall, type QueueCall } from "../internal/calls.js"; +import { registerEventualResource } from "../internal/resources.js"; +import { + isSourceLocation, + type QueueHandlerOptions, + type QueueHandlerSpec, + type QueueSpec, + type SourceLocation, +} from "../internal/service-spec.js"; +import type { DurationSchedule } from "../schedule.js"; +import type { ServiceContext } from "../service.js"; + +export interface QueueHandlerContext { + /** + * Information about the queue. + */ + queue: { + queueName: string; + fifo: boolean; + }; + /** + * Information about the queue handler. + */ + queueHandler: { + queueHandlerName: string; + batch: boolean; + }; + /** + * Information about the containing service. + */ + service: ServiceContext; +} + +export interface QueueHandlerMessageItem { + id: string; + receiptHandle: string; + message: Message; + sent: Date; + receiveCount: number; +} + +export interface QueueHandlerFunction { + /** + * Provides the keys, new value + */ + (item: QueueHandlerMessageItem, context: QueueHandlerContext): + | Promise + | void + | false; +} + +export interface QueueBatchHandlerFunction { + /** + * Provides the keys, new value + */ + (items: QueueHandlerMessageItem[], context: QueueHandlerContext): + | Promise + | void + | { failedMessageIds?: string[] }; +} + +export interface QueueHandler + extends QueueHandlerSpec { + kind: "QueueHandler"; + handler: QueueHandlerFunction; + sourceLocation?: SourceLocation; +} + +export interface QueueBatchHandler + extends QueueHandlerSpec { + kind: "QueueBatchHandler"; + handler: QueueBatchHandlerFunction; + sourceLocation?: SourceLocation; +} + +export interface SendOptions { + delay?: DurationSchedule; +} + +/** + * TODO: support send and delete batch + */ +export interface Queue + extends Omit, "handler" | "message"> { + kind: "Queue"; + handlers: (QueueHandler | QueueBatchHandler)[]; + message?: z.Schema; + sendMessage(message: Message, options?: SendOptions): Promise; + changeMessageVisibility( + receiptHandle: string, + timeout: DurationSchedule + ): Promise; + deleteMessage(receiptHandle: string): Promise; + forEach( + name: Name, + handler: QueueHandlerFunction + ): QueueHandler; + forEach( + name: Name, + options: QueueHandlerOptions, + handler: QueueHandlerFunction + ): QueueHandler; + forEachBatch( + name: Name, + handler: QueueBatchHandlerFunction + ): QueueBatchHandler; + forEachBatch( + name: Name, + options: QueueHandlerOptions, + handler: QueueBatchHandlerFunction + ): QueueBatchHandler; +} + +export interface QueueOptions { + message?: z.Schema; +} + +export function queue( + name: Name, + options?: QueueOptions +): Queue { + const handlers: ( + | QueueHandler + | QueueBatchHandler + )[] = []; + + const queue: Queue = { + kind: "Queue", + handlers, + name, + message: options?.message, + sendMessage(...args) { + return getEventualHook().executeEventualCall( + createCall(CallKind.QueueCall, { + queueName: name, + operation: "sendMessage", + params: args, + }) + ); + }, + changeMessageVisibility(...args) { + return getEventualHook().executeEventualCall( + createCall(CallKind.QueueCall, { + queueName: name, + operation: "changeMessageVisibility", + params: args, + }) + ); + }, + deleteMessage(...args) { + return getEventualHook().executeEventualCall( + createCall(CallKind.QueueCall, { + queueName: name, + operation: "deleteMessage", + params: args, + }) + ); + }, + forEach( + ...args: + | [name: Name, handler: QueueHandlerFunction] + | [ + name: Name, + options: QueueHandlerOptions, + handler: QueueHandlerFunction + ] + | [ + sourceLocation: SourceLocation, + name: Name, + handler: QueueHandlerFunction + ] + | [ + sourceLocation: SourceLocation, + name: Name, + options: QueueHandlerOptions, + handler: QueueHandlerFunction + ] + ) { + const [sourceLocation, handlerName, options, _handler] = + args.length === 4 + ? args + : args.length === 2 + ? [, args[0] as Name, , args[1]] + : isSourceLocation(args[0]) + ? [args[0], args[1] as Name, , args[2]] + : [ + undefined, + ...(args as [ + name: Name, + options: QueueHandlerOptions, + handler: QueueHandlerFunction + ]), + ]; + + if (handlers.some((h) => h.name === handlerName)) { + throw new Error( + `Queue Handler with name ${handlerName} already exists on queue ${name}` + ); + } + + const handler: QueueHandler = { + handler: _handler, + kind: "QueueHandler", + name: handlerName, + sourceLocation, + options, + queueName: name, + }; + + handlers.push(handler as any); + + return handler; + }, + forEachBatch: ( + ...args: + | [name: Name, handler: QueueBatchHandlerFunction] + | [ + name: Name, + options: QueueHandlerOptions, + handler: QueueBatchHandlerFunction + ] + | [ + sourceLocation: SourceLocation, + name: Name, + handler: QueueBatchHandlerFunction + ] + | [ + sourceLocation: SourceLocation, + name: Name, + options: QueueHandlerOptions, + handler: QueueBatchHandlerFunction + ] + ) => { + const [sourceLocation, handlerName, options, _handler] = + args.length === 4 + ? args + : args.length === 2 + ? [, args[0] as Name, , args[1]] + : isSourceLocation(args[0]) + ? [args[0], args[1] as Name, , args[2]] + : [ + undefined, + ...(args as [ + name: Name, + options: QueueHandlerOptions, + handler: QueueBatchHandlerFunction + ]), + ]; + + if (handlers.some((h) => h.name === handlerName)) { + throw new Error( + `Queue Handler with name ${handlerName} already exists on queue ${name}` + ); + } + + const handler: QueueBatchHandler = { + handler: _handler, + kind: "QueueBatchHandler", + name: handlerName, + sourceLocation, + options, + queueName: name, + }; + + handlers.push(handler as any); + + return handler; + }, + }; + + return registerEventualResource("Queue", queue); +} From 0300086287e8da142540db1bc806f8c7ed3ca518 Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Fri, 4 Aug 2023 11:58:50 -0500 Subject: [PATCH 02/20] more queue work --- .../aws-runtime/src/injected/service-spec.ts | 1 + .../core-runtime/src/clients/queue-client.ts | 39 +++++++++++-- .../src/providers/entity-provider.ts | 9 --- .../src/providers/queue-provider.ts | 15 +++++ .../core-runtime/src/stores/entity-store.ts | 2 +- packages/@eventual/core/src/internal/index.ts | 1 + .../core/src/internal/workflow-events.ts | 56 ++++++++++++++++++- 7 files changed, 106 insertions(+), 17 deletions(-) create mode 100644 packages/@eventual/core-runtime/src/providers/queue-provider.ts diff --git a/packages/@eventual/aws-runtime/src/injected/service-spec.ts b/packages/@eventual/aws-runtime/src/injected/service-spec.ts index 0dedbfd33..2cea7b31b 100644 --- a/packages/@eventual/aws-runtime/src/injected/service-spec.ts +++ b/packages/@eventual/aws-runtime/src/injected/service-spec.ts @@ -23,4 +23,5 @@ export default { search: { indices: [], }, + queues: [], } satisfies ServiceSpec; diff --git a/packages/@eventual/core-runtime/src/clients/queue-client.ts b/packages/@eventual/core-runtime/src/clients/queue-client.ts index 76e2a660a..a5d43231e 100644 --- a/packages/@eventual/core-runtime/src/clients/queue-client.ts +++ b/packages/@eventual/core-runtime/src/clients/queue-client.ts @@ -1,11 +1,42 @@ +import type { DurationSchedule, SendOptions } from "@eventual/core"; import { Queue } from "@eventual/core"; import { QueueMethod } from "@eventual/core/internal"; +import type { QueueProvider } from "../providers/queue-provider.js"; -export type QueueClient = { +type QueueClientBase = { [K in keyof Pick]: ( queueName: string, ...args: Parameters ) => ReturnType; -} & { - physicalName: (queueName: string) => string; -}; \ No newline at end of file +}; + +export abstract class QueueClient implements QueueClientBase { + constructor(private queueProvider: QueueProvider) {} + public abstract sendMessage( + queueName: string, + message: any, + options?: SendOptions | undefined + ): Promise; + + public abstract changeMessageVisibility( + queueName: string, + receiptHandle: string, + timeout: DurationSchedule + ): Promise; + + public abstract deleteMessage( + queueName: string, + receiptHandle: string + ): Promise; + + public abstract physicalName: (queueName: string) => string; + + protected getQueue(queueName: string) { + const entity = this.queueProvider.getQueue(queueName); + + if (!entity) { + throw new Error(`Queue ${queueName} was not found.`); + } + return entity; + } +} diff --git a/packages/@eventual/core-runtime/src/providers/entity-provider.ts b/packages/@eventual/core-runtime/src/providers/entity-provider.ts index fe5877be4..ee438ee22 100644 --- a/packages/@eventual/core-runtime/src/providers/entity-provider.ts +++ b/packages/@eventual/core-runtime/src/providers/entity-provider.ts @@ -1,19 +1,10 @@ import type { Entity } from "@eventual/core"; -import type { WorkflowExecutor } from "../workflow/workflow-executor.js"; import { getEventualResource } from "@eventual/core/internal"; export interface EntityProvider { - /** - * Returns an executor which may already be started. - * - * Use {@link WorkflowExecutor}.isStarted to determine if it is already started. - */ getEntity(entityName: string): Entity | undefined; } -/** - * An executor provider that works with an out of memory store. - */ export class GlobalEntityProvider implements EntityProvider { public getEntity(entityName: string): Entity | undefined { return getEventualResource("Entity", entityName); diff --git a/packages/@eventual/core-runtime/src/providers/queue-provider.ts b/packages/@eventual/core-runtime/src/providers/queue-provider.ts new file mode 100644 index 000000000..fdd07ba9b --- /dev/null +++ b/packages/@eventual/core-runtime/src/providers/queue-provider.ts @@ -0,0 +1,15 @@ +import type { Queue } from "@eventual/core"; +import { getEventualResource } from "@eventual/core/internal"; + +export interface QueueProvider { + getQueue(queueName: string): Queue | undefined; +} + +/** + * An executor provider that works with an out of memory store. + */ +export class GlobalQueueProvider implements QueueProvider { + public getQueue(queueName: string): Queue | undefined { + return getEventualResource("Queue", queueName); + } +} diff --git a/packages/@eventual/core-runtime/src/stores/entity-store.ts b/packages/@eventual/core-runtime/src/stores/entity-store.ts index 785dde676..e5e507d6d 100644 --- a/packages/@eventual/core-runtime/src/stores/entity-store.ts +++ b/packages/@eventual/core-runtime/src/stores/entity-store.ts @@ -30,7 +30,7 @@ import { keyHasInlineBetween, type EntityMethod, type KeyDefinition, - type KeyDefinitionPart, + type KeyDefinitionPart } from "@eventual/core/internal"; import type { EntityProvider } from "../providers/entity-provider.js"; diff --git a/packages/@eventual/core/src/internal/index.ts b/packages/@eventual/core/src/internal/index.ts index c5d7839e3..39ed85342 100644 --- a/packages/@eventual/core/src/internal/index.ts +++ b/packages/@eventual/core/src/internal/index.ts @@ -13,3 +13,4 @@ export * from "./signal.js"; export * from "./task.js"; export * from "./util.js"; export * from "./workflow-events.js"; + diff --git a/packages/@eventual/core/src/internal/workflow-events.ts b/packages/@eventual/core/src/internal/workflow-events.ts index 5d136e005..222794e2c 100644 --- a/packages/@eventual/core/src/internal/workflow-events.ts +++ b/packages/@eventual/core/src/internal/workflow-events.ts @@ -6,6 +6,7 @@ import type { BucketMethod, BucketOperation, EntityOperation, + QueueOperation, SearchOperation, } from "./calls.js"; import type { SignalTarget } from "./signal.js"; @@ -43,7 +44,9 @@ export interface CallEventBase< * 15 - Workflow run stated * 16 > 19 - Padding * 20 - Call Event - * 21-49 - Open + * 21-23 - Open + * 24 - Signal Received + * 25-49 - Open * 50 > 79 (current max: 61) - Completed Events * 80 - Workflow Run Completed * 81 > 89 - Padding @@ -59,13 +62,15 @@ export enum WorkflowEventType { ChildWorkflowFailed = 51, EntityRequestFailed = 52, EntityRequestSucceeded = 53, - TransactionRequestFailed = 54, - TransactionRequestSucceeded = 55, + QueueRequestSucceeded = 56, + QueueRequestFailed = 64, SignalReceived = 24, TaskSucceeded = 46, TaskFailed = 57, TaskHeartbeatTimedOut = 58, TimerCompleted = 59, + TransactionRequestFailed = 54, + TransactionRequestSucceeded = 55, WorkflowSucceeded = 95, WorkflowFailed = 96, WorkflowStarted = 10, @@ -78,6 +83,7 @@ export enum WorkflowEventType { export enum WorkflowCallHistoryType { BucketRequest = 0, + QueueRequest = 10, ChildWorkflowScheduled = 1, EntityRequest = 2, EventsEmitted = 3, @@ -107,6 +113,7 @@ export type WorkflowCallHistoryEvent = | SearchRequest | EntityRequest | EventsEmitted + | QueueRequest | SignalSent | TaskScheduled | TimerScheduled @@ -122,6 +129,8 @@ export type CompletionEvent = | ChildWorkflowSucceeded | EntityRequestFailed | EntityRequestSucceeded + | QueueRequestSucceeded + | QueueRequestFailed | SignalReceived | SearchRequestSucceeded | SearchRequestFailed @@ -146,6 +155,8 @@ export const isCompletionEvent = /* @__PURE__ */ or( isChildWorkflowSucceeded, isEntityRequestFailed, isEntityRequestSucceeded, + isQueueRequestFailed, + isQueueRequestSucceeded, isSignalReceived, isTaskSucceeded, isTaskFailed, @@ -299,6 +310,45 @@ export function isTaskHeartbeatTimedOut( return event.type === WorkflowEventType.TaskHeartbeatTimedOut; } +export interface QueueRequest + extends CallEventBase { + type: WorkflowCallHistoryType.QueueRequest; + operation: QueueOperation; +} + +export interface QueueRequestSucceeded + extends CallEventResultBase { + name?: string; + operation: QueueOperation["operation"]; + result: any; +} + +export interface QueueRequestFailed + extends CallEventResultBase { + operation: QueueOperation["operation"]; + name?: string; + error: string; + message: string; +} + +export function isQueueRequest( + event: WorkflowCallHistoryEvent +): event is QueueRequest { + return event.type === WorkflowCallHistoryType.QueueRequest; +} + +export function isQueueRequestSucceeded( + event: WorkflowEvent +): event is QueueRequestSucceeded { + return event.type === WorkflowEventType.QueueRequestSucceeded; +} + +export function isQueueRequestFailed( + event: WorkflowEvent +): event is QueueRequestFailed { + return event.type === WorkflowEventType.QueueRequestFailed; +} + export interface EntityRequest extends CallEventBase { operation: EntityOperation; From 6d8a753bd58e3a427ee102c4c87ab412e4e57d90 Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Wed, 9 Aug 2023 13:53:11 -0500 Subject: [PATCH 03/20] complete runtime changes --- packages/@eventual/cli/src/commands/replay.ts | 3 +- .../src/call-executors/queue-call-executor.ts | 6 +- .../core-runtime/src/handlers/orchestrator.ts | 5 ++ .../src/handlers/transaction-worker.ts | 1 + .../src/workflow/call-eventual-factory.ts | 4 +- .../src/workflow/call-executor.ts | 9 ++- .../queue-call.ts | 74 +++++++++++++++++++ 7 files changed, 98 insertions(+), 4 deletions(-) create mode 100644 packages/@eventual/core-runtime/src/workflow/call-executors-and-factories/queue-call.ts diff --git a/packages/@eventual/cli/src/commands/replay.ts b/packages/@eventual/cli/src/commands/replay.ts index 0fe010c28..67acaadf9 100644 --- a/packages/@eventual/cli/src/commands/replay.ts +++ b/packages/@eventual/cli/src/commands/replay.ts @@ -89,12 +89,13 @@ export const replay = (yargs: Argv) => const executor = new WorkflowExecutor( workflow, events, - // TODO: these properties should come from the history + // TODO: these properties should come from the history https://github.com/functionless/eventual/issues/416 new AllPropertyRetriever({ ServiceClient: serviceClient, ServiceName: serviceName ?? unsupportedPropertyRetriever, OpenSearchClient: unsupportedPropertyRetriever, BucketPhysicalName: unsupportedPropertyRetriever, + QueuePhysicalName: unsupportedPropertyRetriever, ServiceSpec: unsupportedPropertyRetriever, ServiceType: ServiceType.OrchestratorWorker, ServiceUrl: serviceData.apiEndpoint ?? unsupportedPropertyRetriever, diff --git a/packages/@eventual/core-runtime/src/call-executors/queue-call-executor.ts b/packages/@eventual/core-runtime/src/call-executors/queue-call-executor.ts index d721f91a8..a7f99527b 100644 --- a/packages/@eventual/core-runtime/src/call-executors/queue-call-executor.ts +++ b/packages/@eventual/core-runtime/src/call-executors/queue-call-executor.ts @@ -5,6 +5,10 @@ import { QueueClient } from "../clients/queue-client.js"; export class QueueCallExecutor implements CallExecutor { constructor(private queueClient: QueueClient) {} public execute(call: QueueCall): Promise { - return this.queueClient[call.operation](call.queueName, ...call.params); + return this.queueClient[call.operation]( + call.queueName, + // @ts-ignore + ...call.params + ); } } diff --git a/packages/@eventual/core-runtime/src/handlers/orchestrator.ts b/packages/@eventual/core-runtime/src/handlers/orchestrator.ts index 8f7d08edf..127aab45c 100644 --- a/packages/@eventual/core-runtime/src/handlers/orchestrator.ts +++ b/packages/@eventual/core-runtime/src/handlers/orchestrator.ts @@ -65,6 +65,8 @@ import { WorkflowCallExecutor } from "../workflow/call-executor.js"; import { createEvent } from "../workflow/events.js"; import { isExecutionId, parseWorkflowName } from "../workflow/execution.js"; import { WorkflowExecutor } from "../workflow/workflow-executor.js"; +import { QueuePhysicalNamePropertyRetriever } from "../property-retrievers/queue-name-property-retriever.js"; +import { QueueClient } from "../clients/queue-client.js"; export interface OrchestratorResult { /** @@ -90,11 +92,13 @@ export function createOrchestrator( const unsupportedProperty = new UnsupportedPropertyRetriever( "Workflow Orchestrator" ); + // TODO: load these from history when running past executions https://github.com/functionless/eventual/issues/416 const propertyRetriever = new AllPropertyRetriever({ BucketPhysicalName: new BucketPhysicalNamePropertyRetriever( deps.bucketStore ), OpenSearchClient: unsupportedProperty, + QueuePhysicalName: new QueuePhysicalNamePropertyRetriever(deps.queueClient), ServiceClient: unsupportedProperty, ServiceName: deps.serviceName, ServiceSpec: unsupportedProperty, @@ -137,6 +141,7 @@ interface OrchestratorDependencies { executorProvider: ExecutorProvider; logAgent?: LogAgent; metricsClient?: MetricsClient; + queueClient: QueueClient; serviceName: string; timerClient: TimerClient; workflowClient: WorkflowClient; diff --git a/packages/@eventual/core-runtime/src/handlers/transaction-worker.ts b/packages/@eventual/core-runtime/src/handlers/transaction-worker.ts index 2fef5192c..ba82f9473 100644 --- a/packages/@eventual/core-runtime/src/handlers/transaction-worker.ts +++ b/packages/@eventual/core-runtime/src/handlers/transaction-worker.ts @@ -38,6 +38,7 @@ export function createTransactionWorker( const propertyRetriever = new AllPropertyRetriever({ BucketPhysicalName: unsupportedPropertyRetriever, OpenSearchClient: unsupportedPropertyRetriever, + QueuePhysicalName: unsupportedPropertyRetriever, ServiceClient: unsupportedPropertyRetriever, ServiceName: props.serviceName, ServiceSpec: unsupportedPropertyRetriever, diff --git a/packages/@eventual/core-runtime/src/workflow/call-eventual-factory.ts b/packages/@eventual/core-runtime/src/workflow/call-eventual-factory.ts index c1afaaedd..c461d842a 100644 --- a/packages/@eventual/core-runtime/src/workflow/call-eventual-factory.ts +++ b/packages/@eventual/core-runtime/src/workflow/call-eventual-factory.ts @@ -19,6 +19,7 @@ import { TaskCallEventualFactory } from "./call-executors-and-factories/task-cal import { TransactionCallEventualFactory } from "./call-executors-and-factories/transaction-call.js"; import { UnsupportedEventualFactory } from "./call-executors-and-factories/unsupported.js"; import type { EventualDefinition } from "./eventual-definition.js"; +import { QueueCallEventualFactory } from "./call-executors-and-factories/queue-call.js"; export interface ResolveEventualFunction { (seq: number, result: Result): void; @@ -75,9 +76,10 @@ export function createDefaultEventualFactory(): AllWorkflowEventualFactory { ExpectSignalCall: new ExpectSignalFactory(), GetExecutionCall: unsupportedFactory, InvokeTransactionCall: new TransactionCallEventualFactory(), - SignalHandlerCall: new RegisterSignalHandlerCallFactory(), + QueueCall: new QueueCallEventualFactory(), SearchCall: new SearchCallEventualFactory(), SendSignalCall: new SendSignalEventualFactory(), + SignalHandlerCall: new RegisterSignalHandlerCallFactory(), StartWorkflowCall: unsupportedFactory, TaskCall: new TaskCallEventualFactory(), TaskRequestCall: unsupportedFactory, // TODO: support task requests (succeed, fail, heartbeat) diff --git a/packages/@eventual/core-runtime/src/workflow/call-executor.ts b/packages/@eventual/core-runtime/src/workflow/call-executor.ts index e899c66cf..c0391bb6f 100644 --- a/packages/@eventual/core-runtime/src/workflow/call-executor.ts +++ b/packages/@eventual/core-runtime/src/workflow/call-executor.ts @@ -26,6 +26,8 @@ import { SimpleWorkflowExecutorAdaptor } from "./call-executors-and-factories/si import { createTransactionWorkflowQueueExecutor } from "./call-executors-and-factories/transaction-call.js"; import { UnsupportedWorkflowCallExecutor } from "./call-executors-and-factories/unsupported.js"; import { ChildWorkflowCallWorkflowExecutor } from "./call-executors-and-factories/child-workflow-call.js"; +import { QueueClient } from "../clients/queue-client.js"; +import { createQueueCallWorkflowCallExecutor } from "./call-executors-and-factories/queue-call.js"; interface WorkflowCallExecutorDependencies { bucketStore: BucketStore; @@ -33,6 +35,7 @@ interface WorkflowCallExecutorDependencies { eventClient: EventClient; openSearchClient: OpenSearchClient; executionQueueClient: ExecutionQueueClient; + queueClient: QueueClient; taskClient: TaskClient; timerClient: TimerClient; transactionClient: TransactionClient; @@ -68,7 +71,10 @@ export function createDefaultWorkflowCallExecutor( deps.transactionClient, deps.executionQueueClient ), - SignalHandlerCall: noOpExecutor, // signal handlers do not generate events + QueueCall: createQueueCallWorkflowCallExecutor( + deps.queueClient, + deps.executionQueueClient + ), SearchCall: createSearchWorkflowQueueExecutor( deps.openSearchClient, deps.executionQueueClient @@ -76,6 +82,7 @@ export function createDefaultWorkflowCallExecutor( SendSignalCall: new SendSignalWorkflowCallExecutor( deps.executionQueueClient ), + SignalHandlerCall: noOpExecutor, // signal handlers do not generate events TaskCall: new TaskCallWorkflowExecutor(deps.taskClient), TaskRequestCall: unsupportedExecutor, // TODO: add support for task heartbeat, success, and failure to the workflow ChildWorkflowCall: workflowClientExecutor, diff --git a/packages/@eventual/core-runtime/src/workflow/call-executors-and-factories/queue-call.ts b/packages/@eventual/core-runtime/src/workflow/call-executors-and-factories/queue-call.ts new file mode 100644 index 000000000..9fa0b2621 --- /dev/null +++ b/packages/@eventual/core-runtime/src/workflow/call-executors-and-factories/queue-call.ts @@ -0,0 +1,74 @@ +import { EventualError } from "@eventual/core"; +import { + QueueCall, + QueueRequestFailed, + QueueRequestSucceeded, + WorkflowCallHistoryType, + WorkflowEventType, +} from "@eventual/core/internal"; +import { Result, normalizeError } from "../../result.js"; +import { EventualFactory } from "../call-eventual-factory.js"; +import { EventualDefinition, Trigger } from "../eventual-definition.js"; +import { QueueClient } from "../../clients/queue-client.js"; +import { WorkflowTaskQueueExecutorAdaptor } from "./task-queue-executor-adaptor.js"; +import { QueueCallExecutor } from "../../call-executors/queue-call-executor.js"; +import { ExecutionQueueClient } from "../../clients/execution-queue-client.js"; +import { createEvent } from "../events.js"; + +export function createQueueCallWorkflowCallExecutor( + queueClient: QueueClient, + executionQueueClient: ExecutionQueueClient +) { + return new WorkflowTaskQueueExecutorAdaptor( + new QueueCallExecutor(queueClient), + executionQueueClient, + (call: QueueCall, result, { seq, executionTime }) => { + return createEvent( + { + type: WorkflowEventType.QueueRequestSucceeded, + operation: call.operation, + result: result as any, + seq, + name: call.queueName, + }, + executionTime + ); + }, + (call: QueueCall, err, { executionTime, seq }) => { + return createEvent( + { + type: WorkflowEventType.QueueRequestFailed, + operation: call.operation, + seq, + ...normalizeError(err), + }, + executionTime + ); + } + ); +} + +export class QueueCallEventualFactory implements EventualFactory { + public initializeEventual(call: QueueCall): EventualDefinition { + return { + triggers: [ + Trigger.onWorkflowEvent( + WorkflowEventType.QueueRequestSucceeded, + (event) => Result.resolved(event.result) + ), + Trigger.onWorkflowEvent(WorkflowEventType.QueueRequestFailed, (event) => + Result.failed(new EventualError(event.error, event.message)) + ), + ], + createCallEvent: (seq) => ({ + type: WorkflowCallHistoryType.QueueRequest, + seq, + operation: { + operation: call.operation, + params: call.params, + queueName: call.queueName, + }, + }), + }; + } +} From 20f6a07077cc8876d140722895f0185174b298a2 Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Wed, 9 Aug 2023 16:34:06 -0500 Subject: [PATCH 04/20] add fifo to core and runtime --- packages/@eventual/compiler/src/ast-util.ts | 22 +- .../@eventual/compiler/src/eventual-infer.ts | 29 +- .../src/call-executors/queue-call-executor.ts | 10 +- .../core-runtime/src/clients/queue-client.ts | 18 +- .../src/providers/queue-provider.ts | 6 +- .../queue-call.ts | 12 +- packages/@eventual/core/src/internal/calls.ts | 44 ++- .../@eventual/core/src/internal/resources.ts | 2 + .../core/src/internal/service-spec.ts | 13 +- packages/@eventual/core/src/queue/fifo.ts | 353 ++++++++++++++++++ packages/@eventual/core/src/queue/queue.ts | 58 ++- 11 files changed, 511 insertions(+), 56 deletions(-) diff --git a/packages/@eventual/compiler/src/ast-util.ts b/packages/@eventual/compiler/src/ast-util.ts index 33b1f79ef..03b1b721d 100644 --- a/packages/@eventual/compiler/src/ast-util.ts +++ b/packages/@eventual/compiler/src/ast-util.ts @@ -74,13 +74,33 @@ export function isBucketHandlerMemberCall(call: CallExpression): boolean { if (c.type === "MemberExpression") { if (isId(c.property, "on")) { // bucket.stream(events, "handlerName", async () => { }) - // bucket.stream(events, ""handlerName", options, async () => { }) + // bucket.stream(events, "handlerName", options, async () => { }) return call.arguments.length === 3 || call.arguments.length === 4; } } return false; } +/** + * A heuristic for identifying a {@link CallExpression} that is a call to an `on` handler. + * + * 1. must be a call to a MemberExpression matching to `.on(events, name, impl | props, impl)`. + * 2. must have 3 or 4 arguments. + */ +export function isQueueHandlerForEachMemberCall(call: CallExpression): boolean { + const c = call.callee; + if (c.type === "MemberExpression") { + if (isId(c.property, "forEach") || isId(c.property, "forEachBatch")) { + // queue.forEach("handlerName", async () => { }) + // queue.forEach("handlerName", options, async () => { }) + // queue.forEachBatch("handlerName", async () => { }) + // queue.forEachBatch("handlerName", options, async () => { }) + return call.arguments.length === 2 || call.arguments.length === 3; + } + } + return false; +} + /** * A heuristic for identifying a {@link CallExpression} that is a call to an `subscription` handler. * diff --git a/packages/@eventual/compiler/src/eventual-infer.ts b/packages/@eventual/compiler/src/eventual-infer.ts index ae5f6cca8..51ea83a13 100644 --- a/packages/@eventual/compiler/src/eventual-infer.ts +++ b/packages/@eventual/compiler/src/eventual-infer.ts @@ -6,10 +6,12 @@ */ import { generateSchema } from "@anatine/zod-openapi"; import { - BucketNotificationHandlerSpec, - CommandSpec, - ServiceSpec, getEventualResources, + type BucketNotificationHandlerSpec, + type CommandSpec, + type QueueHandlerSpec, + type QueueSpec, + type ServiceSpec, } from "@eventual/core/internal"; import { CallExpression, @@ -31,6 +33,7 @@ import { isCommandCall, isEntityStreamMemberCall, isOnEventCall, + isQueueHandlerForEachMemberCall, isSubscriptionCall, isTaskCall, } from "./ast-util.js"; @@ -168,6 +171,25 @@ export function inferFromMemory(openApi: ServiceSpec["openApi"]): ServiceSpec { }) ), }, + queues: Array.from(getEventualResources("Queue").values()).map( + (q) => + ({ + name: q.name, + handlers: q.handlers.map( + (h) => + ({ + name: h.name, + queueName: q.name, + options: h.options, + batch: h.batch, + fifo: h.fifo, + sourceLocation: h.sourceLocation, + } satisfies QueueHandlerSpec) + ), + message: q.message ? generateSchema(q.message) : undefined, + fifo: q.fifo, + } satisfies QueueSpec) + ), }; } @@ -253,6 +275,7 @@ export class InferVisitor extends Visitor { isTaskCall, isEntityStreamMemberCall, isBucketHandlerMemberCall, + isQueueHandlerForEachMemberCall, ].some((op) => op(call)) ) { this.didMutate = true; diff --git a/packages/@eventual/core-runtime/src/call-executors/queue-call-executor.ts b/packages/@eventual/core-runtime/src/call-executors/queue-call-executor.ts index a7f99527b..8c267dfb3 100644 --- a/packages/@eventual/core-runtime/src/call-executors/queue-call-executor.ts +++ b/packages/@eventual/core-runtime/src/call-executors/queue-call-executor.ts @@ -1,12 +1,16 @@ -import { QueueCall } from "@eventual/core/internal"; +import { QueueCall, isQueueOperationOfType } from "@eventual/core/internal"; import type { CallExecutor } from "../call-executor.js"; import { QueueClient } from "../clients/queue-client.js"; export class QueueCallExecutor implements CallExecutor { constructor(private queueClient: QueueClient) {} public execute(call: QueueCall): Promise { - return this.queueClient[call.operation]( - call.queueName, + const operation = call.operation; + if (isQueueOperationOfType("sendMessage", operation)) { + return this.queueClient.sendMessage(operation); + } + return this.queueClient[operation.operation]( + call.operation.queueName, // @ts-ignore ...call.params ); diff --git a/packages/@eventual/core-runtime/src/clients/queue-client.ts b/packages/@eventual/core-runtime/src/clients/queue-client.ts index a5d43231e..57df926f2 100644 --- a/packages/@eventual/core-runtime/src/clients/queue-client.ts +++ b/packages/@eventual/core-runtime/src/clients/queue-client.ts @@ -1,21 +1,27 @@ -import type { DurationSchedule, SendOptions } from "@eventual/core"; +import type { DurationSchedule, FifoQueue } from "@eventual/core"; import { Queue } from "@eventual/core"; -import { QueueMethod } from "@eventual/core/internal"; +import { + QueueMethod, + QueueSendMessageOperation, +} from "@eventual/core/internal"; import type { QueueProvider } from "../providers/queue-provider.js"; type QueueClientBase = { - [K in keyof Pick]: ( + [K in keyof Pick>]: ( queueName: string, ...args: Parameters ) => ReturnType; +} & { + sendMessage: ( + operation: QueueSendMessageOperation + ) => ReturnType; }; export abstract class QueueClient implements QueueClientBase { constructor(private queueProvider: QueueProvider) {} + public abstract sendMessage( - queueName: string, - message: any, - options?: SendOptions | undefined + operation: QueueSendMessageOperation ): Promise; public abstract changeMessageVisibility( diff --git a/packages/@eventual/core-runtime/src/providers/queue-provider.ts b/packages/@eventual/core-runtime/src/providers/queue-provider.ts index fdd07ba9b..a89ef7c27 100644 --- a/packages/@eventual/core-runtime/src/providers/queue-provider.ts +++ b/packages/@eventual/core-runtime/src/providers/queue-provider.ts @@ -1,15 +1,15 @@ -import type { Queue } from "@eventual/core"; +import type { FifoQueue, Queue } from "@eventual/core"; import { getEventualResource } from "@eventual/core/internal"; export interface QueueProvider { - getQueue(queueName: string): Queue | undefined; + getQueue(queueName: string): FifoQueue | Queue | undefined; } /** * An executor provider that works with an out of memory store. */ export class GlobalQueueProvider implements QueueProvider { - public getQueue(queueName: string): Queue | undefined { + public getQueue(queueName: string): FifoQueue | Queue | undefined { return getEventualResource("Queue", queueName); } } diff --git a/packages/@eventual/core-runtime/src/workflow/call-executors-and-factories/queue-call.ts b/packages/@eventual/core-runtime/src/workflow/call-executors-and-factories/queue-call.ts index 9fa0b2621..423c554dc 100644 --- a/packages/@eventual/core-runtime/src/workflow/call-executors-and-factories/queue-call.ts +++ b/packages/@eventual/core-runtime/src/workflow/call-executors-and-factories/queue-call.ts @@ -26,10 +26,10 @@ export function createQueueCallWorkflowCallExecutor( return createEvent( { type: WorkflowEventType.QueueRequestSucceeded, - operation: call.operation, + operation: call.operation.operation, result: result as any, seq, - name: call.queueName, + name: call.operation.queueName, }, executionTime ); @@ -38,7 +38,7 @@ export function createQueueCallWorkflowCallExecutor( return createEvent( { type: WorkflowEventType.QueueRequestFailed, - operation: call.operation, + operation: call.operation.operation, seq, ...normalizeError(err), }, @@ -63,11 +63,7 @@ export class QueueCallEventualFactory implements EventualFactory { createCallEvent: (seq) => ({ type: WorkflowCallHistoryType.QueueRequest, seq, - operation: { - operation: call.operation, - params: call.params, - queueName: call.queueName, - }, + operation: call.operation, }), }; } diff --git a/packages/@eventual/core/src/internal/calls.ts b/packages/@eventual/core/src/internal/calls.ts index dd7e35380..1456688c4 100644 --- a/packages/@eventual/core/src/internal/calls.ts +++ b/packages/@eventual/core/src/internal/calls.ts @@ -7,6 +7,7 @@ import type { } from "../entity/entity.js"; import type { EventEnvelope } from "../event.js"; import type { Execution, ExecutionHandle } from "../execution.js"; +import { FifoContentBasedDeduplication, FifoQueue } from "../queue/fifo.js"; import type { Queue } from "../queue/queue.js"; import type { DurationSchedule, Schedule } from "../schedule.js"; import type { SearchIndex } from "../search/search-index.js"; @@ -235,22 +236,41 @@ export type QueueMethod = Exclude< "forEach" | "forEachBatch" | undefined >; -export type QueueCall = CallBase< - CallKind.QueueCall, - ReturnType -> & - QueueOperation; +export interface QueueCall + extends CallBase> { + operation: QueueOperation; +} -export type QueueOperation = { - operation: Op; +interface QueueOperationBase { queueName: string; - params: Parameters; -}; + operation: Op; +} + +export type QueueSendMessageOperation = QueueOperationBase<"sendMessage"> & { + delay?: DurationSchedule; + message: any; +} & ( + | { + fifo: true; + messageGroupId: string; + messageDeduplicationId: string | FifoContentBasedDeduplication; + } + | { + fifo: false; + } + ); + +export type QueueOperation = + Op extends "sendMessage" + ? QueueSendMessageOperation + : QueueOperationBase & { + params: Parameters; + }; -export function isQueueCallType( +export function isQueueOperationOfType( op: Op, - operation: QueueCall -): operation is QueueCall { + operation: QueueOperation +): operation is QueueOperation { return operation.operation === op; } diff --git a/packages/@eventual/core/src/internal/resources.ts b/packages/@eventual/core/src/internal/resources.ts index 28bcb0887..dd52fdfce 100644 --- a/packages/@eventual/core/src/internal/resources.ts +++ b/packages/@eventual/core/src/internal/resources.ts @@ -2,6 +2,7 @@ import type { Bucket } from "../bucket.js"; import type { Entity } from "../entity/entity.js"; import type { Event } from "../event.js"; import type { AnyCommand } from "../http/command.js"; +import type { FifoQueue } from "../queue/fifo.js"; import type { Queue } from "../queue/queue.js"; import type { SearchIndex } from "../search/search-index.js"; import type { Subscription } from "../subscription.js"; @@ -19,6 +20,7 @@ type Resource = | Task | Transaction | Queue + | FifoQueue | Workflow; type ResourceKind = Resource["kind"]; diff --git a/packages/@eventual/core/src/internal/service-spec.ts b/packages/@eventual/core/src/internal/service-spec.ts index 9eaaaf57b..774b75c34 100644 --- a/packages/@eventual/core/src/internal/service-spec.ts +++ b/packages/@eventual/core/src/internal/service-spec.ts @@ -295,9 +295,12 @@ export interface TransactionSpec { */ export interface QueueHandlerOptions extends FunctionRuntimeProps { /** - * Max batch size. Between 1 and 1000. + * Max batch size. * - * @default: 100 + * Queue - Between 1 and 10000. + * Fifo Queue - Between 1 and 10 + * + * @default: 100 (Queue) 10 (Fifo Queue). */ batchSize?: number; /** @@ -311,13 +314,15 @@ export interface QueueHandlerOptions extends FunctionRuntimeProps { export interface QueueHandlerSpec { name: Name; queueName: string; - options?: EntityStreamOptions; + options?: QueueHandlerOptions; + batch: boolean; + fifo: boolean; sourceLocation?: SourceLocation; } export interface QueueSpec { name: Name; handlers: QueueHandlerSpec[]; + fifo: boolean; message?: openapi.SchemaObject; - attributes?: openapi.SchemaObject; } diff --git a/packages/@eventual/core/src/queue/fifo.ts b/packages/@eventual/core/src/queue/fifo.ts index e69de29bb..f8d015dad 100644 --- a/packages/@eventual/core/src/queue/fifo.ts +++ b/packages/@eventual/core/src/queue/fifo.ts @@ -0,0 +1,353 @@ +import { CallKind, createCall, type QueueCall } from "../internal/calls.js"; +import { registerEventualResource } from "../internal/resources.js"; +import { + isSourceLocation, + type QueueHandlerOptions, + type SourceLocation, +} from "../internal/service-spec.js"; +import type { + Queue, + QueueBatchHandler, + QueueBatchHandlerFunction, + QueueHandler, + QueueHandlerFunction, + QueueHandlerMessageItem, + QueueOptions, + QueueSendMessageOptions, +} from "./queue.js"; + +export interface FifoQueueHandlerMessageItem + extends QueueHandlerMessageItem { + messageGroupId: string; + sequenceNumber: string; + messageDeduplicationId: string; +} + +/** + * Assertion that content based deduplication is on. + * + * Can be overridden when calling `sendMessage`. + */ +export interface FifoContentBasedDeduplication { + contentBasedDeduplication: true; +} + +export type FifoQueueHandlerFunction = QueueHandlerFunction< + Message, + FifoQueueHandlerMessageItem +>; + +export type FifoQueueBatchHandlerFunction = + QueueBatchHandlerFunction>; + +export interface FifoQueueHandler + extends Omit, "handler" | "fifo"> { + handler: FifoQueueHandlerFunction; + fifo: true; +} + +export interface FifoQueueBatchHandler< + Name extends string = string, + Message = any +> extends Omit, "handler" | "fifo"> { + handler: FifoQueueBatchHandlerFunction; + fifo: true; +} + +export interface FifoQueueSendOptions extends QueueSendMessageOptions { + messageGroupId?: string; + messageDeduplicationId?: string; +} + +/** + * TODO: support send and delete batch + */ +export interface FifoQueue + extends Omit< + Queue, + "sendMessage" | "handlers" | "forEach" | "forEachBatch" + > { + handlers: ( + | FifoQueueHandler + | FifoQueueBatchHandler + )[]; + fifo: true; + sendMessage(message: Message, options?: FifoQueueSendOptions): Promise; + forEach( + name: Name, + handler: FifoQueueHandlerFunction + ): FifoQueueHandler; + forEach( + name: Name, + options: QueueHandlerOptions, + handler: FifoQueueHandlerFunction + ): FifoQueueHandler; + forEachBatch( + name: Name, + handler: FifoQueueBatchHandlerFunction + ): FifoQueueBatchHandler; + forEachBatch( + name: Name, + options: QueueHandlerOptions, + handler: FifoQueueBatchHandlerFunction + ): FifoQueueBatchHandler; +} + +export type MessageIdField = { + [K in keyof Message]: K extends string + ? // only include attributes that extend string or number + Message[K] extends string + ? K + : never + : never; +}[keyof Message]; + +export type FifoQueueMessagePropertyReference = + | MessageIdField + | ((message: Message) => string); + +export interface FifoQueueOptions< + Message = any, + MessageGroupId extends + | FifoQueueMessagePropertyReference + | undefined = FifoQueueMessagePropertyReference | undefined, + MessageDeduplicationId extends + | FifoQueueMessagePropertyReference + | undefined + | FifoContentBasedDeduplication = + | FifoQueueMessagePropertyReference + | undefined + | FifoContentBasedDeduplication +> extends QueueOptions { + /** + * The field or function used to compute the `messageGroupId`. + * + * The `messageGroupId` determines how messages are grouped in the Fifo Queue. + * Messages with the same Id will be sent to both forEach and forEachBatch handlers in the order retrieved and cannot progress until + * the first one succeeds. + * + * * Field Name - provide a field to use for the `messageGroupId`. This field must be required and must contain a string value. + * * Getter Function - provide a (synchronous) function to compute the group id from a message on send. + * * undefined - `messageGroupId` will be required on `sendMessage`. + * + * Can be overridden during `sendMessage` by providing a messageGroupId there. + * + * @default undefined - must be set during `sendMessage`. + */ + messageGroupId?: MessageGroupId; + /** + * A field or setting for content deduplication. + * + * * Field Name - provide a field to use for content deduplication. This field must be required and must contain a string value. + * * Getter Function - provide a (synchronous) function to compute the deduplication id from a message on send. + * * `CONTENT_BASED_DEDUPE` - use the content of the message to compute the deduplication id. + * + * Any of these setting can be overridden during send message by providing a messageDeduplicationId there. + */ + messageDeduplicationId?: MessageDeduplicationId; +} + +export function fifoQueue( + name: Name, + options?: FifoQueueOptions +): FifoQueue { + const handlers: ( + | FifoQueueHandler + | FifoQueueBatchHandler + )[] = []; + + const messageGroupIdReference = options?.messageGroupId; + const messageDedupeIdReference = options?.messageDeduplicationId; + + const queue: FifoQueue = { + kind: "Queue", + handlers, + name, + fifo: true, + message: options?.message, + sendMessage(message, sendOptions) { + const messageGroupId = + sendOptions?.messageGroupId ?? messageGroupIdReference + ? typeof messageGroupIdReference === "string" + ? message[messageGroupIdReference] + : messageGroupIdReference?.(message) + : undefined; + if (!messageGroupId || typeof messageGroupId !== "string") { + throw new Error( + "Message Group Id must be provided and must be a non-empty string" + ); + } + const messageDeduplicationId = + sendOptions?.messageDeduplicationId ?? messageDedupeIdReference + ? typeof messageDedupeIdReference === "string" + ? message[messageDedupeIdReference] + : typeof messageDedupeIdReference === "function" + ? messageDedupeIdReference?.(message) + : messageDedupeIdReference + : undefined; + if ( + !messageDeduplicationId || + !( + typeof messageDeduplicationId === "string" || + (typeof messageDeduplicationId === "object" && + "contentBasedDeduplication" in messageDeduplicationId) + ) + ) { + throw new Error( + "Message Deduplication Id must be provided and a non-empty string or set to { contentBasedDeduplication: true }" + ); + } + return getEventualHook().executeEventualCall( + createCall(CallKind.QueueCall, { + operation: { + queueName: name, + operation: "sendMessage", + fifo: true, + message, + delay: sendOptions?.delay, + messageGroupId, + messageDeduplicationId, + }, + }) + ); + }, + changeMessageVisibility(...args) { + return getEventualHook().executeEventualCall( + createCall(CallKind.QueueCall, { + operation: { + queueName: name, + operation: "changeMessageVisibility", + params: args, + }, + }) + ); + }, + deleteMessage(...args) { + return getEventualHook().executeEventualCall( + createCall(CallKind.QueueCall, { + operation: { + queueName: name, + operation: "deleteMessage", + params: args, + }, + }) + ); + }, + forEach( + ...args: + | [name: Name, handler: FifoQueueHandlerFunction] + | [ + name: Name, + options: QueueHandlerOptions, + handler: FifoQueueHandlerFunction + ] + | [ + sourceLocation: SourceLocation, + name: Name, + handler: FifoQueueHandlerFunction + ] + | [ + sourceLocation: SourceLocation, + name: Name, + options: QueueHandlerOptions, + handler: FifoQueueHandlerFunction + ] + ) { + const [sourceLocation, handlerName, options, _handler] = + args.length === 4 + ? args + : args.length === 2 + ? [, args[0] as Name, , args[1]] + : isSourceLocation(args[0]) + ? [args[0], args[1] as Name, , args[2]] + : [ + undefined, + ...(args as [ + name: Name, + options: QueueHandlerOptions, + handler: FifoQueueHandlerFunction + ]), + ]; + + if (handlers.some((h) => h.name === handlerName)) { + throw new Error( + `Queue Handler with name ${handlerName} already exists on queue ${name}` + ); + } + + const handler: FifoQueueHandler = { + handler: _handler, + kind: "QueueHandler", + name: handlerName, + sourceLocation, + options, + batch: false, + fifo: true, + queueName: name, + }; + + handlers.push(handler as any); + + return handler; + }, + forEachBatch: ( + ...args: + | [name: Name, handler: FifoQueueBatchHandlerFunction] + | [ + name: Name, + options: QueueHandlerOptions, + handler: FifoQueueBatchHandlerFunction + ] + | [ + sourceLocation: SourceLocation, + name: Name, + handler: FifoQueueBatchHandlerFunction + ] + | [ + sourceLocation: SourceLocation, + name: Name, + options: QueueHandlerOptions, + handler: FifoQueueBatchHandlerFunction + ] + ) => { + const [sourceLocation, handlerName, options, _handler] = + args.length === 4 + ? args + : args.length === 2 + ? [, args[0] as Name, , args[1]] + : isSourceLocation(args[0]) + ? [args[0], args[1] as Name, , args[2]] + : [ + undefined, + ...(args as [ + name: Name, + options: QueueHandlerOptions, + handler: FifoQueueBatchHandlerFunction + ]), + ]; + + if (handlers.some((h) => h.name === handlerName)) { + throw new Error( + `Queue Handler with name ${handlerName} already exists on queue ${name}` + ); + } + + const handler: FifoQueueBatchHandler = { + handler: _handler, + kind: "QueueBatchHandler", + name: handlerName, + sourceLocation, + options, + batch: true, + fifo: true, + queueName: name, + }; + + handlers.push(handler as any); + + return handler; + }, + }; + + return registerEventualResource("Queue", queue); +} diff --git a/packages/@eventual/core/src/queue/queue.ts b/packages/@eventual/core/src/queue/queue.ts index ec969b601..4cf430409 100644 --- a/packages/@eventual/core/src/queue/queue.ts +++ b/packages/@eventual/core/src/queue/queue.ts @@ -40,21 +40,27 @@ export interface QueueHandlerMessageItem { receiveCount: number; } -export interface QueueHandlerFunction { +export interface QueueHandlerFunction< + Message = any, + MessageItem extends QueueHandlerMessageItem = QueueHandlerMessageItem +> { /** * Provides the keys, new value */ - (item: QueueHandlerMessageItem, context: QueueHandlerContext): + (item: MessageItem, context: QueueHandlerContext): | Promise | void | false; } -export interface QueueBatchHandlerFunction { +export interface QueueBatchHandlerFunction< + Message = any, + MessageItem extends QueueHandlerMessageItem = QueueHandlerMessageItem +> { /** * Provides the keys, new value */ - (items: QueueHandlerMessageItem[], context: QueueHandlerContext): + (items: MessageItem[], context: QueueHandlerContext): | Promise | void | { failedMessageIds?: string[] }; @@ -65,6 +71,8 @@ export interface QueueHandler kind: "QueueHandler"; handler: QueueHandlerFunction; sourceLocation?: SourceLocation; + fifo: false; + batch: false; } export interface QueueBatchHandler @@ -72,9 +80,11 @@ export interface QueueBatchHandler kind: "QueueBatchHandler"; handler: QueueBatchHandlerFunction; sourceLocation?: SourceLocation; + fifo: false; + batch: true; } -export interface SendOptions { +export interface QueueSendMessageOptions { delay?: DurationSchedule; } @@ -86,7 +96,10 @@ export interface Queue kind: "Queue"; handlers: (QueueHandler | QueueBatchHandler)[]; message?: z.Schema; - sendMessage(message: Message, options?: SendOptions): Promise; + sendMessage( + message: Message, + options?: QueueSendMessageOptions + ): Promise; changeMessageVisibility( receiptHandle: string, timeout: DurationSchedule @@ -129,31 +142,40 @@ export function queue( kind: "Queue", handlers, name, + fifo: false, message: options?.message, - sendMessage(...args) { + sendMessage(message, options) { return getEventualHook().executeEventualCall( createCall(CallKind.QueueCall, { - queueName: name, - operation: "sendMessage", - params: args, + operation: { + queueName: name, + operation: "sendMessage", + fifo: false, + message, + delay: options?.delay, + }, }) ); }, changeMessageVisibility(...args) { return getEventualHook().executeEventualCall( createCall(CallKind.QueueCall, { - queueName: name, - operation: "changeMessageVisibility", - params: args, + operation: { + queueName: name, + operation: "changeMessageVisibility", + params: args, + }, }) ); }, deleteMessage(...args) { return getEventualHook().executeEventualCall( createCall(CallKind.QueueCall, { - queueName: name, - operation: "deleteMessage", - params: args, + operation: { + queueName: name, + operation: "deleteMessage", + params: args, + }, }) ); }, @@ -205,6 +227,8 @@ export function queue( name: handlerName, sourceLocation, options, + batch: false, + fifo: false, queueName: name, }; @@ -260,6 +284,8 @@ export function queue( name: handlerName, sourceLocation, options, + batch: true, + fifo: false, queueName: name, }; From e2f76d5612fde1b42195b262b44a6df9321e1230 Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Thu, 10 Aug 2023 08:19:46 -0500 Subject: [PATCH 05/20] add aws queue client --- .../aws-runtime/src/clients/queue-client.ts | 106 ++++++++++++++++++ packages/@eventual/aws-runtime/src/utils.ts | 14 +++ .../core-runtime/src/clients/index.ts | 1 + .../core-runtime/src/clients/queue-client.ts | 2 +- .../core-runtime/src/providers/index.ts | 1 + packages/@eventual/core/src/queue/fifo.ts | 9 +- 6 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 packages/@eventual/aws-runtime/src/clients/queue-client.ts diff --git a/packages/@eventual/aws-runtime/src/clients/queue-client.ts b/packages/@eventual/aws-runtime/src/clients/queue-client.ts new file mode 100644 index 000000000..7fc61b8f0 --- /dev/null +++ b/packages/@eventual/aws-runtime/src/clients/queue-client.ts @@ -0,0 +1,106 @@ +import { + ChangeMessageVisibilityCommand, + DeleteMessageCommand, + SendMessageCommand, + type SQSClient, +} from "@aws-sdk/client-sqs"; +import { + isFifoContentBasedDeduplication, + type DurationSchedule, +} from "@eventual/core"; +import { + computeDurationSeconds, + getLazy, + type LazyValue, + QueueClient, + type QueueProvider, +} from "@eventual/core-runtime"; +import type { QueueSendMessageOperation } from "@eventual/core/internal"; +import { queueServiceQueueName } from "../utils.js"; + +export interface QueueRuntimeOverrides { + /** + * Override the queue name of the queue. + */ + queueName?: string; +} + +export interface AWSQueueClientProps { + sqs: SQSClient; + queueProvider: QueueProvider; + awsAccount: LazyValue; + awsRegion: LazyValue; + serviceName: LazyValue; + queueOverrides: LazyValue>; +} + +export class AWSQueueClient extends QueueClient { + constructor(private props: AWSQueueClientProps) { + super(props.queueProvider); + } + + public override async sendMessage( + operation: QueueSendMessageOperation + ): Promise { + await this.props.sqs.send( + new SendMessageCommand({ + MessageBody: operation.message, + QueueUrl: this.physicalQueueUrl(operation.queueName), + DelaySeconds: operation.delay + ? computeDurationSeconds(operation.delay) + : undefined, + MessageDeduplicationId: + // message deduplication is only supported for FIFO queues + // if fifo, deduplication id is required unless the definition asserts content based deduplication is on. + !operation.fifo || + isFifoContentBasedDeduplication(operation.messageDeduplicationId) + ? undefined + : operation.messageDeduplicationId, + MessageGroupId: operation.fifo ? operation.messageGroupId : undefined, + }) + ); + } + + public override async changeMessageVisibility( + queueName: string, + receiptHandle: string, + timeout: DurationSchedule + ): Promise { + await this.props.sqs.send( + new ChangeMessageVisibilityCommand({ + QueueUrl: this.physicalQueueUrl(queueName), + ReceiptHandle: receiptHandle, + VisibilityTimeout: computeDurationSeconds(timeout), + }) + ); + } + + public override async deleteMessage( + queueName: string, + receiptHandle: string + ): Promise { + await this.props.sqs.send( + new DeleteMessageCommand({ + QueueUrl: this.physicalQueueUrl(queueName), + ReceiptHandle: receiptHandle, + }) + ); + } + + public physicalQueueUrl(queueName: string) { + return `https://sqs.${getLazy( + this.props.awsRegion + )}.amazonaws.com/${getLazy(this.props.awsAccount)}/${this.physicalName( + queueName + )}`; + } + + public override physicalName(queueName: string) { + const overrides = getLazy(this.props.queueOverrides); + const nameOverride = overrides[queueName]?.queueName; + return ( + nameOverride ?? + queueServiceQueueName(getLazy(this.props.serviceName), queueName) + ); + } +} diff --git a/packages/@eventual/aws-runtime/src/utils.ts b/packages/@eventual/aws-runtime/src/utils.ts index cc2c6f438..1b322fa3d 100644 --- a/packages/@eventual/aws-runtime/src/utils.ts +++ b/packages/@eventual/aws-runtime/src/utils.ts @@ -258,6 +258,10 @@ export function bucketServiceBucketSuffix(bucketName: string) { return `bucket-${bucketName}`; } +export function queueServiceQueueSuffix(queueName: string) { + return `queue-${queueName}`; +} + export function taskServiceFunctionName( serviceName: string, taskId: string @@ -282,6 +286,16 @@ export function bucketServiceBucketName( return serviceBucketName(serviceName, bucketServiceBucketSuffix(bucketName)); } +/** + * Note: a queue's name can be overridden by the user. + */ +export function queueServiceQueueName( + serviceName: string, + queueName: string +): string { + return serviceFunctionName(serviceName, queueServiceQueueSuffix(queueName)); +} + /** * Valid lambda function names contains letters, numbers, dash, or underscore and no spaces. */ diff --git a/packages/@eventual/core-runtime/src/clients/index.ts b/packages/@eventual/core-runtime/src/clients/index.ts index 77549d9a5..81f08fd99 100644 --- a/packages/@eventual/core-runtime/src/clients/index.ts +++ b/packages/@eventual/core-runtime/src/clients/index.ts @@ -3,6 +3,7 @@ export * from "./execution-queue-client.js"; export * from "./logs-client.js"; export * from "./metrics-client.js"; export * from "./open-search-client.js"; +export * from "./queue-client.js"; export * from "./runtime-service-clients.js"; export * from "./task-client.js"; export * from "./timer-client.js"; diff --git a/packages/@eventual/core-runtime/src/clients/queue-client.ts b/packages/@eventual/core-runtime/src/clients/queue-client.ts index 57df926f2..a5d69d3de 100644 --- a/packages/@eventual/core-runtime/src/clients/queue-client.ts +++ b/packages/@eventual/core-runtime/src/clients/queue-client.ts @@ -35,7 +35,7 @@ export abstract class QueueClient implements QueueClientBase { receiptHandle: string ): Promise; - public abstract physicalName: (queueName: string) => string; + public abstract physicalName(queueName: string): string; protected getQueue(queueName: string) { const entity = this.queueProvider.getQueue(queueName); diff --git a/packages/@eventual/core-runtime/src/providers/index.ts b/packages/@eventual/core-runtime/src/providers/index.ts index ef2029583..b9963be4b 100644 --- a/packages/@eventual/core-runtime/src/providers/index.ts +++ b/packages/@eventual/core-runtime/src/providers/index.ts @@ -1,5 +1,6 @@ export * from "./entity-provider.js"; export * from "./executor-provider.js"; +export * from "./queue-provider.js"; export * from "./subscription-provider.js"; export * from "./task-provider.js"; export * from "./workflow-provider.js"; diff --git a/packages/@eventual/core/src/queue/fifo.ts b/packages/@eventual/core/src/queue/fifo.ts index f8d015dad..2461e9a10 100644 --- a/packages/@eventual/core/src/queue/fifo.ts +++ b/packages/@eventual/core/src/queue/fifo.ts @@ -32,6 +32,12 @@ export interface FifoContentBasedDeduplication { contentBasedDeduplication: true; } +export function isFifoContentBasedDeduplication( + value: any +): value is FifoContentBasedDeduplication { + return value && typeof value === "object" && value.contentBasedDeduplication; +} + export type FifoQueueHandlerFunction = QueueHandlerFunction< Message, FifoQueueHandlerMessageItem @@ -189,8 +195,7 @@ export function fifoQueue( !messageDeduplicationId || !( typeof messageDeduplicationId === "string" || - (typeof messageDeduplicationId === "object" && - "contentBasedDeduplication" in messageDeduplicationId) + isFifoContentBasedDeduplication(messageDeduplicationId) ) ) { throw new Error( From 37b5d2a03b35bbcc0f5587d501189ece73691b03 Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Thu, 10 Aug 2023 10:02:10 -0500 Subject: [PATCH 06/20] runtime queue handler worker --- .../src/handlers/entity-stream-worker.ts | 39 +++--- .../src/handlers/queue-handler-worker.ts | 122 ++++++++++++++++++ packages/@eventual/core-runtime/src/utils.ts | 59 +++++++++ .../core/src/internal/service-type.ts | 7 +- packages/@eventual/core/src/queue/fifo.ts | 4 + 5 files changed, 204 insertions(+), 27 deletions(-) create mode 100644 packages/@eventual/core-runtime/src/handlers/queue-handler-worker.ts diff --git a/packages/@eventual/core-runtime/src/handlers/entity-stream-worker.ts b/packages/@eventual/core-runtime/src/handlers/entity-stream-worker.ts index af3dfa6f1..11a0acb33 100644 --- a/packages/@eventual/core-runtime/src/handlers/entity-stream-worker.ts +++ b/packages/@eventual/core-runtime/src/handlers/entity-stream-worker.ts @@ -1,7 +1,7 @@ import type { EntityStreamContext, EntityStreamItem } from "@eventual/core"; import { ServiceType, getEventualResource } from "@eventual/core/internal"; import { normalizeCompositeKey } from "../stores/entity-store.js"; -import { getLazy, promiseAllSettledPartitioned } from "../utils.js"; +import { getLazy, groupedPromiseAllSettled } from "../utils.js"; import { createEventualWorker, type WorkerIntrinsicDeps } from "./worker.js"; export interface EntityStreamWorker { @@ -42,34 +42,25 @@ export function createEntityStreamWorker( return { failedItemIds: result?.failedItemIds ?? [] }; } else { - const itemsByKey: Record[]> = {}; - items.forEach((item) => { - const normalizedKey = normalizeCompositeKey(entity, item.key); - const serializedKey = JSON.stringify(normalizedKey); - (itemsByKey[serializedKey] ??= []).push(item); - }); - - const results = await promiseAllSettledPartitioned( - Object.entries(itemsByKey), - async ([, itemGroup]) => { - for (const i in itemGroup) { - const item = itemGroup[i]!; - try { - const result = await streamHandler.handler(item, context); - // if the handler doesn't fail and doesn't return false, continue - if (result !== false) { - continue; - } - } catch {} - // if the handler fails or returns false, return the rest of the items - return itemGroup.slice(Number(i)).map((i) => i.id); + const groupResults = await groupedPromiseAllSettled( + items, + (item) => { + const normalizedKey = normalizeCompositeKey(entity, item.key); + return JSON.stringify(normalizedKey); + }, + async (item) => { + const result = streamHandler.handler(item, context); + if (result === false) { + throw new Error("Handler failed"); } - return []; + return result; } ); return { - failedItemIds: results.fulfilled.flatMap((s) => s[1]), + failedItemIds: Object.values(groupResults).flatMap((g) => + g.rejected.map(([item]) => item.id) + ), }; } } diff --git a/packages/@eventual/core-runtime/src/handlers/queue-handler-worker.ts b/packages/@eventual/core-runtime/src/handlers/queue-handler-worker.ts new file mode 100644 index 000000000..80d51425e --- /dev/null +++ b/packages/@eventual/core-runtime/src/handlers/queue-handler-worker.ts @@ -0,0 +1,122 @@ +import { + isFifoQueue, + type FifoQueueHandler, + type FifoQueueHandlerMessageItem, + type QueueHandlerContext, + type QueueHandlerMessageItem, +} from "@eventual/core"; +import { ServiceType, getEventualResource } from "@eventual/core/internal"; +import { + getLazy, + groupedPromiseAllSettled, + promiseAllSettledPartitioned, +} from "../utils.js"; +import { createEventualWorker, type WorkerIntrinsicDeps } from "./worker.js"; + +export type QueueHandlerDependencies = WorkerIntrinsicDeps; + +export interface QueueHandler { + ( + queueName: string, + handlerName: string, + items: (FifoQueueHandlerMessageItem | QueueHandlerMessageItem)[] + ): Promise; +} + +export function createQueueHandlerWorker( + dependencies: QueueHandlerDependencies +): QueueHandler { + return createEventualWorker( + { serviceType: ServiceType.QueueHandlerWorker, ...dependencies }, + async (queueName, handlerName, items) => { + const queue = getEventualResource("Queue", queueName); + if (!queue) throw new Error(`Queue ${queueName} does not exist`); + if (isFifoQueue(queue)) { + const handler = queue.handlers.find((h) => h.name === handlerName); + + if (!handler) throw new Error(`Handler ${handlerName} does not exist`); + + const context: QueueHandlerContext = { + queue: { queueName, fifo: queue.fifo }, + queueHandler: { batch: handler.batch, queueHandlerName: handlerName }, + service: { + serviceName: getLazy(dependencies.serviceName), + serviceUrl: getLazy(dependencies.serviceUrl), + }, + }; + + if (handler.kind === "QueueBatchHandler") { + const result = await handler.handler( + items as FifoQueueHandlerMessageItem[], + context + ); + if (result?.failedMessageIds && result.failedMessageIds.length > 0) { + return { failedMessageIds: result.failedMessageIds }; + } + return undefined; + } else { + // fifo queue needs to maintain order of messages and fail preceding messages. + // in AWS, failing a message out of order will cause an error. + // ex: ([A, B]; A fails and B does not, AWS will throw an error on delete). + const groupResults = await groupedPromiseAllSettled( + items as FifoQueueHandlerMessageItem[], + (item) => item.messageGroupId, + async (item) => { + const result = (handler as FifoQueueHandler).handler( + item, + context + ); + if (result === false) { + throw new Error("Handler reported failure"); + } + return result; + } + ); + + return { + failedMessageIds: Object.values(groupResults).flatMap((g) => + g.rejected.map(([item]) => item.id) + ), + }; + } + } else { + const handler = queue.handlers.find((h) => h.name === handlerName); + + if (!handler) { + throw new Error(`Handler ${handlerName} does not exist`); + } + + const context: QueueHandlerContext = { + queue: { queueName, fifo: queue.fifo }, + queueHandler: { batch: handler.batch, queueHandlerName: handlerName }, + service: { + serviceName: getLazy(dependencies.serviceName), + serviceUrl: getLazy(dependencies.serviceUrl), + }, + }; + + if (handler?.kind === "QueueBatchHandler") { + const result = await handler.handler( + items as QueueHandlerMessageItem[], + context + ); + + if (result?.failedMessageIds && result.failedMessageIds.length > 0) { + return { failedMessageIds: result.failedMessageIds ?? [] }; + } + return undefined; + } else { + // normal queue handler doesn't have a concept of order, pass all messages in any order. + const results = await promiseAllSettledPartitioned( + items, + async (item) => handler.handler(item, context) + ); + + return { + failedMessageIds: results.rejected.map(([item]) => item.id), + }; + } + } + } + ); +} diff --git a/packages/@eventual/core-runtime/src/utils.ts b/packages/@eventual/core-runtime/src/utils.ts index 0dba173d6..597b69656 100644 --- a/packages/@eventual/core-runtime/src/utils.ts +++ b/packages/@eventual/core-runtime/src/utils.ts @@ -44,6 +44,65 @@ export async function promiseAllSettledPartitioned( }; } +/** + * Groups the items based on the group function and then executes the groups in parallel, but the items within a group in order. + * + * When one item in a group failed, fail the rest of the group which have yet to succeed. + */ +export async function groupedPromiseAllSettled( + items: I[], + group: (item: I) => string, + handler: (item: I) => Promise | R +): Promise< + Record< + string, + { + fulfilled: (readonly [I, Awaited])[]; + rejected: (readonly [I, any])[]; + } + > +> { + const itemsByKey: Record = {}; + items.forEach((item) => { + const key = group(item); + (itemsByKey[key] ??= []).push(item); + }); + + const results = await promiseAllSettledPartitioned( + Object.entries(itemsByKey), + async ([, itemGroup]) => { + const fulfilled: [I, Awaited][] = []; + for (const i in itemGroup) { + const item = itemGroup[i]!; + try { + const result = await handler(item); + // if the handler doesn't fail and doesn't return false, continue + fulfilled.push([item, result]); + continue; + } catch (err) { + return { + fulfilled, + rejected: [ + [item, err] as const, + ...itemGroup + .slice(Number(i) + 1) + .map((i) => [i, "cascading failure"] as const), + ], + }; + } + } + return { + fulfilled, + rejected: [] as [I, any][], + }; + } + ); + + return Object.fromEntries( + results.fulfilled.map(([[group], r]) => [group, r]) + ); +} + export function groupBy( items: T[], extract: (item: T) => string diff --git a/packages/@eventual/core/src/internal/service-type.ts b/packages/@eventual/core/src/internal/service-type.ts index b83b67913..1cb53d3c0 100644 --- a/packages/@eventual/core/src/internal/service-type.ts +++ b/packages/@eventual/core/src/internal/service-type.ts @@ -1,11 +1,12 @@ import { PropertyKind, createEventualProperty } from "./properties.js"; export enum ServiceType { + BucketNotificationHandlerWorker = "BucketNotificationHandlerWorker", CommandWorker = "CommandWorker", - Subscription = "Subscription", - OrchestratorWorker = "OrchestratorWorker", EntityStreamWorker = "EntityStreamWorker", - BucketNotificationHandlerWorker = "BucketNotificationHandlerWorker", + OrchestratorWorker = "OrchestratorWorker", + QueueHandlerWorker = "QueueHandlerWorker", + Subscription = "Subscription", TaskWorker = "TaskWorker", TransactionWorker = "TransactionWorker", } diff --git a/packages/@eventual/core/src/queue/fifo.ts b/packages/@eventual/core/src/queue/fifo.ts index 2461e9a10..4101fb7ad 100644 --- a/packages/@eventual/core/src/queue/fifo.ts +++ b/packages/@eventual/core/src/queue/fifo.ts @@ -65,6 +65,10 @@ export interface FifoQueueSendOptions extends QueueSendMessageOptions { messageDeduplicationId?: string; } +export function isFifoQueue(queue: Queue | FifoQueue): queue is FifoQueue { + return queue.fifo; +} + /** * TODO: support send and delete batch */ From 335b03d33b94a258ce5751b283ee2b60020803d0 Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Thu, 10 Aug 2023 11:21:19 -0500 Subject: [PATCH 07/20] queue handler worker aws --- packages/@eventual/aws-runtime/src/create.ts | 13 ++++ packages/@eventual/aws-runtime/src/env.ts | 14 ++++ .../src/handlers/queue-handler-worker.ts | 67 +++++++++++++++++++ .../core-runtime/src/handlers/index.ts | 1 + 4 files changed, 95 insertions(+) create mode 100644 packages/@eventual/aws-runtime/src/handlers/queue-handler-worker.ts diff --git a/packages/@eventual/aws-runtime/src/create.ts b/packages/@eventual/aws-runtime/src/create.ts index 386b76b1a..46632d059 100644 --- a/packages/@eventual/aws-runtime/src/create.ts +++ b/packages/@eventual/aws-runtime/src/create.ts @@ -12,6 +12,7 @@ import { ExecutionQueueClient, ExecutionStore, GlobalEntityProvider, + GlobalQueueProvider, GlobalTaskProvider, GlobalWorkflowProvider, LogAgent, @@ -37,6 +38,7 @@ import { AWSExecutionStore } from "./stores/execution-store.js"; import { AWSTaskStore } from "./stores/task-store.js"; import { AWSOpenSearchClient } from "./clients/opensearch-client.js"; import { defaultProvider } from "@aws-sdk/credential-provider-node"; +import { AWSQueueClient } from "./clients/queue-client.js"; /** * Client creators to be used by the lambda functions. @@ -210,6 +212,17 @@ export const createTaskClient = /* @__PURE__ */ memoize( }) ); +export const createQueueClient = memoize(() => { + return new AWSQueueClient({ + sqs: sqs(), + awsAccount: env.awsAccount, + serviceName: env.serviceName, + awsRegion: env.awsRegion, + queueOverrides: env.queueOverrides, + queueProvider: new GlobalQueueProvider(), + }); +}); + export const createExecutionHistoryStateStore = /* @__PURE__ */ memoize( ({ executionHistoryBucket }: { executionHistoryBucket?: string } = {}) => new AWSExecutionHistoryStateStore({ diff --git a/packages/@eventual/aws-runtime/src/env.ts b/packages/@eventual/aws-runtime/src/env.ts index 34505a62d..bcad9f26b 100644 --- a/packages/@eventual/aws-runtime/src/env.ts +++ b/packages/@eventual/aws-runtime/src/env.ts @@ -1,6 +1,7 @@ import { LogLevel } from "@eventual/core"; import { assertNonNull } from "@eventual/core/internal"; import { BucketRuntimeOverrides } from "./stores/bucket-store.js"; +import { QueueRuntimeOverrides } from "./clients/queue-client.js"; export const ENV_NAMES = { SERVICE_NAME: "EVENTUAL_SERVICE_NAME", @@ -22,10 +23,13 @@ export const ENV_NAMES = { DEFAULT_LOG_LEVEL: "EVENTUAL_LOG_LEVEL", ENTITY_NAME: "EVENTUAL_ENTITY_NAME", ENTITY_STREAM_NAME: "EVENTUAL_ENTITY_STREAM_NAME", + QUEUE_NAME: "EVENTUAL_QUEUE_NAME", + QUEUE_HANDLER_NAME: "EVENTUAL_QUEUE_HANDLER_NAME", BUCKET_NAME: "EVENTUAL_BUCKET_NAME", BUCKET_HANDLER_NAME: "EVENTUAL_BUCKET_HANDLER_NAME", TRANSACTION_WORKER_ARN: "EVENTUAL_TRANSACTION_WORKER_ARN", BUCKET_OVERRIDES: "EVENTUAL_BUCKET_OVERRIDES", + QUEUE_OVERRIDES: "EVENTUAL_QUEUE_OVERRIDES", } as const; export function tryGetEnv(name: string) { @@ -35,6 +39,7 @@ export function tryGetEnv(name: string) { ) as T; } +export const awsAccount = () => tryGetEnv("AWS_ACCOUNT_ID"); export const awsRegion = () => tryGetEnv("AWS_REGION"); export const serviceName = () => tryGetEnv(ENV_NAMES.SERVICE_NAME); export const openSearchEndpoint = () => @@ -64,6 +69,8 @@ export const entityName = () => tryGetEnv(ENV_NAMES.ENTITY_NAME); export const entityStreamName = () => tryGetEnv(ENV_NAMES.ENTITY_STREAM_NAME); export const bucketName = () => tryGetEnv(ENV_NAMES.BUCKET_NAME); export const bucketHandlerName = () => tryGetEnv(ENV_NAMES.BUCKET_HANDLER_NAME); +export const queueName = () => tryGetEnv(ENV_NAMES.QUEUE_NAME); +export const queueHandlerName = () => tryGetEnv(ENV_NAMES.QUEUE_HANDLER_NAME); export const transactionWorkerArn = () => tryGetEnv(ENV_NAMES.TRANSACTION_WORKER_ARN); export const bucketOverrides = () => { @@ -73,3 +80,10 @@ export const bucketOverrides = () => { BucketRuntimeOverrides >; }; +export const queueOverrides = () => { + const queueOverridesString = process.env[ENV_NAMES.QUEUE_OVERRIDES] ?? "{}"; + return JSON.parse(queueOverridesString) as Record< + string, + QueueRuntimeOverrides + >; +}; diff --git a/packages/@eventual/aws-runtime/src/handlers/queue-handler-worker.ts b/packages/@eventual/aws-runtime/src/handlers/queue-handler-worker.ts new file mode 100644 index 000000000..3c310e6d3 --- /dev/null +++ b/packages/@eventual/aws-runtime/src/handlers/queue-handler-worker.ts @@ -0,0 +1,67 @@ +import serviceSpec from "@eventual/injected/spec"; +// the user's entry point will register streams as a side effect. +import "@eventual/injected/entry"; + +import type { + FifoQueueHandlerMessageItem, + QueueHandlerMessageItem, +} from "@eventual/core"; +import { createQueueHandlerWorker, getLazy } from "@eventual/core-runtime"; +import type { SQSBatchItemFailure, SQSHandler } from "aws-lambda"; +import { + createBucketStore, + createEntityStore, + createOpenSearchClient, + createQueueClient, + createServiceClient, +} from "../create.js"; +import { + queueHandlerName, + queueName, + serviceName, + serviceUrl, +} from "../env.js"; + +const worker = createQueueHandlerWorker({ + queueClient: createQueueClient(), + bucketStore: createBucketStore(), + entityStore: createEntityStore(), + openSearchClient: await createOpenSearchClient(), + serviceClient: createServiceClient({}), + serviceSpec, + serviceName, + serviceUrl, +}); + +export default (async (event) => { + const items: (FifoQueueHandlerMessageItem | QueueHandlerMessageItem)[] = + event.Records.map( + (r) => + ({ + id: r.messageId, + message: r.body, + sequenceNumber: r.attributes.SequenceNumber, + messageDeduplicationId: r.attributes.MessageDeduplicationId, + messageGroupId: r.attributes.MessageGroupId, + receiptHandle: r.receiptHandle, + receiveCount: Number(r.attributes.ApproximateReceiveCount), + sent: new Date(r.attributes.SentTimestamp), + } satisfies FifoQueueHandlerMessageItem | QueueHandlerMessageItem) + ); + const result = await worker( + getLazy(queueName), + getLazy(queueHandlerName), + items + ); + if (result) { + return { + batchItemFailures: result.failedMessageIds.map( + (id) => + ({ + itemIdentifier: id, + } satisfies SQSBatchItemFailure) + ), + }; + } + return undefined; +}); diff --git a/packages/@eventual/core-runtime/src/handlers/index.ts b/packages/@eventual/core-runtime/src/handlers/index.ts index c9b4c376e..16126ffba 100644 --- a/packages/@eventual/core-runtime/src/handlers/index.ts +++ b/packages/@eventual/core-runtime/src/handlers/index.ts @@ -2,6 +2,7 @@ export * from "./bucket-handler-worker.js"; export * from "./command-worker.js"; export * from "./entity-stream-worker.js"; export * from "./orchestrator.js"; +export * from "./queue-handler-worker.js"; export * from "./schedule-forwarder.js"; export * from "./subscription-worker.js"; export * from "./task-fallback-handler.js"; From 9c054a940bade9d8600dd7a8be42d04c5cebe64a Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Thu, 10 Aug 2023 11:25:47 -0500 Subject: [PATCH 08/20] update aws handlers with queue client --- .../@eventual/aws-runtime/src/handlers/apig-command-worker.ts | 1 + .../aws-runtime/src/handlers/bucket-handler-worker.ts | 2 ++ packages/@eventual/aws-runtime/src/handlers/command-worker.ts | 2 ++ .../@eventual/aws-runtime/src/handlers/entity-stream-worker.ts | 2 ++ packages/@eventual/aws-runtime/src/handlers/orchestrator.ts | 3 +++ .../@eventual/aws-runtime/src/handlers/subscription-worker.ts | 2 ++ .../aws-runtime/src/handlers/system-command-handler.ts | 2 ++ packages/@eventual/aws-runtime/src/handlers/task-worker.ts | 2 ++ packages/@eventual/core-runtime/src/handlers/worker.ts | 1 - 9 files changed, 16 insertions(+), 1 deletion(-) diff --git a/packages/@eventual/aws-runtime/src/handlers/apig-command-worker.ts b/packages/@eventual/aws-runtime/src/handlers/apig-command-worker.ts index b09af9841..d07bd9c82 100644 --- a/packages/@eventual/aws-runtime/src/handlers/apig-command-worker.ts +++ b/packages/@eventual/aws-runtime/src/handlers/apig-command-worker.ts @@ -47,6 +47,7 @@ export function createApiGCommandWorker({ bucketStore: deps.bucketStore, entityStore: deps.entityStore, openSearchClient: deps.openSearchClient, + queueClient: deps.queueClient, serviceClient, serviceSpec, serviceName, diff --git a/packages/@eventual/aws-runtime/src/handlers/bucket-handler-worker.ts b/packages/@eventual/aws-runtime/src/handlers/bucket-handler-worker.ts index 94912953d..3632e05a9 100644 --- a/packages/@eventual/aws-runtime/src/handlers/bucket-handler-worker.ts +++ b/packages/@eventual/aws-runtime/src/handlers/bucket-handler-worker.ts @@ -12,6 +12,7 @@ import { createBucketStore, createEntityStore, createOpenSearchClient, + createQueueClient, createServiceClient, } from "../create.js"; import { @@ -26,6 +27,7 @@ const worker = createBucketNotificationHandlerWorker({ entityStore: createEntityStore(), openSearchClient: await createOpenSearchClient(), serviceClient: createServiceClient({}), + queueClient: createQueueClient(), serviceName, serviceSpec, serviceUrl, diff --git a/packages/@eventual/aws-runtime/src/handlers/command-worker.ts b/packages/@eventual/aws-runtime/src/handlers/command-worker.ts index 609300c9d..45bffb555 100644 --- a/packages/@eventual/aws-runtime/src/handlers/command-worker.ts +++ b/packages/@eventual/aws-runtime/src/handlers/command-worker.ts @@ -6,6 +6,7 @@ import { createEntityStore, createEventClient, createOpenSearchClient, + createQueueClient, createServiceClient, createTransactionClient, } from "../create.js"; @@ -22,6 +23,7 @@ export default createApiGCommandWorker({ bucketStore: createBucketStore(), entityStore: createEntityStore(), openSearchClient: await createOpenSearchClient(), + queueClient: createQueueClient(), // the service client, spec, and service url will be created at runtime, using a computed uri from the apigateway request // pulls the service url from the request instead of env variables to reduce the circular dependency between commands and the gateway. serviceClientBuilder: (serviceUrl) => diff --git a/packages/@eventual/aws-runtime/src/handlers/entity-stream-worker.ts b/packages/@eventual/aws-runtime/src/handlers/entity-stream-worker.ts index 2a9ca2d9b..38a5aa7f4 100644 --- a/packages/@eventual/aws-runtime/src/handlers/entity-stream-worker.ts +++ b/packages/@eventual/aws-runtime/src/handlers/entity-stream-worker.ts @@ -18,6 +18,7 @@ import { createBucketStore, createEntityStore, createOpenSearchClient, + createQueueClient, createServiceClient, } from "../create.js"; import { @@ -34,6 +35,7 @@ const worker = createEntityStreamWorker({ bucketStore: createBucketStore(), entityStore: createEntityStore(), openSearchClient: await createOpenSearchClient(), + queueClient: createQueueClient(), serviceClient: createServiceClient({}), serviceSpec, serviceName, diff --git a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts index abd7626ae..e48b8f00a 100644 --- a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts +++ b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts @@ -17,6 +17,7 @@ import { createExecutionQueueClient, createLogAgent, createOpenSearchClient, + createQueueClient, createTaskClient, createTimerClient, createTransactionClient, @@ -41,6 +42,7 @@ const orchestrate = createOrchestrator({ bucketStore: createBucketStore(), entityStore: createEntityStore(), eventClient: createEventClient(), + queueClient: createQueueClient(), executionQueueClient: createExecutionQueueClient(), openSearchClient: await createOpenSearchClient(), taskClient: createTaskClient(), @@ -48,6 +50,7 @@ const orchestrate = createOrchestrator({ transactionClient: createTransactionClient(), workflowClient: createWorkflowClient(), }), + queueClient: createQueueClient(), serviceName: serviceName(), timerClient: createTimerClient(), workflowClient: createWorkflowClient(), diff --git a/packages/@eventual/aws-runtime/src/handlers/subscription-worker.ts b/packages/@eventual/aws-runtime/src/handlers/subscription-worker.ts index 551ab0ac6..0ff79cc51 100644 --- a/packages/@eventual/aws-runtime/src/handlers/subscription-worker.ts +++ b/packages/@eventual/aws-runtime/src/handlers/subscription-worker.ts @@ -11,6 +11,7 @@ import { createEntityStore, createEventClient, createOpenSearchClient, + createQueueClient, createServiceClient, createTransactionClient, } from "../create.js"; @@ -25,6 +26,7 @@ export const processEvent = createSubscriptionWorker({ eventClient: createEventClient(), transactionClient: createTransactionClient(), }), + queueClient: createQueueClient(), serviceSpec, subscriptionProvider: new GlobalSubscriptionProvider(), serviceName, diff --git a/packages/@eventual/aws-runtime/src/handlers/system-command-handler.ts b/packages/@eventual/aws-runtime/src/handlers/system-command-handler.ts index 16d23ba8b..c5415c634 100644 --- a/packages/@eventual/aws-runtime/src/handlers/system-command-handler.ts +++ b/packages/@eventual/aws-runtime/src/handlers/system-command-handler.ts @@ -22,6 +22,7 @@ import { createExecutionHistoryStore, createExecutionQueueClient, createExecutionStore, + createQueueClient, createTaskClient, createTransactionClient, createWorkflowClient, @@ -34,6 +35,7 @@ function systemCommandWorker( ): APIGatewayProxyHandlerV2 { return createApiGCommandWorker({ bucketStore: createBucketStore(), + queueClient: createQueueClient(), entityStore: undefined, openSearchClient: undefined, serviceSpec, diff --git a/packages/@eventual/aws-runtime/src/handlers/task-worker.ts b/packages/@eventual/aws-runtime/src/handlers/task-worker.ts index e771a829f..f69f157ea 100644 --- a/packages/@eventual/aws-runtime/src/handlers/task-worker.ts +++ b/packages/@eventual/aws-runtime/src/handlers/task-worker.ts @@ -17,6 +17,7 @@ import { createExecutionStore, createLogAgent, createOpenSearchClient, + createQueueClient, createServiceClient, createTaskClient, createTaskStore, @@ -31,6 +32,7 @@ const worker = createTaskWorker({ eventClient: createEventClient(), openSearchClient: await createOpenSearchClient(), executionQueueClient: createExecutionQueueClient(), + queueClient: createQueueClient(), logAgent: createLogAgent(), metricsClient: AWSMetricsClient, // partially uses the runtime clients and partially uses the http client diff --git a/packages/@eventual/core-runtime/src/handlers/worker.ts b/packages/@eventual/core-runtime/src/handlers/worker.ts index 189f97912..e728de50e 100644 --- a/packages/@eventual/core-runtime/src/handlers/worker.ts +++ b/packages/@eventual/core-runtime/src/handlers/worker.ts @@ -35,7 +35,6 @@ export interface WorkerIntrinsicDeps { serviceName: string | LazyValue; serviceSpec: ServiceSpec | undefined; serviceUrl: string | LazyValue; - serviceUrls?: (string | LazyValue)[]; } type AllExecutorOverrides = { From b43b1c5296a6b044af830205a26e2a7e8de4af2d Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Thu, 10 Aug 2023 21:28:19 -0500 Subject: [PATCH 09/20] local support --- .../core-runtime/src/clients/queue-client.ts | 2 +- .../src/handlers/queue-handler-worker.ts | 4 +- .../src/local/clients/queue-client.ts | 208 ++++++++++++++++++ .../core-runtime/src/local/local-container.ts | 61 ++++- .../src/local/local-environment.ts | 33 ++- .../core/src/internal/service-spec.ts | 1 + packages/@eventual/core/src/queue/fifo.ts | 1 + packages/@eventual/core/src/queue/queue.ts | 7 + packages/@eventual/testing/src/environment.ts | 34 +++ 9 files changed, 342 insertions(+), 9 deletions(-) create mode 100644 packages/@eventual/core-runtime/src/local/clients/queue-client.ts diff --git a/packages/@eventual/core-runtime/src/clients/queue-client.ts b/packages/@eventual/core-runtime/src/clients/queue-client.ts index a5d69d3de..79f9e6dab 100644 --- a/packages/@eventual/core-runtime/src/clients/queue-client.ts +++ b/packages/@eventual/core-runtime/src/clients/queue-client.ts @@ -18,7 +18,7 @@ type QueueClientBase = { }; export abstract class QueueClient implements QueueClientBase { - constructor(private queueProvider: QueueProvider) {} + constructor(protected queueProvider: QueueProvider) {} public abstract sendMessage( operation: QueueSendMessageOperation diff --git a/packages/@eventual/core-runtime/src/handlers/queue-handler-worker.ts b/packages/@eventual/core-runtime/src/handlers/queue-handler-worker.ts index 80d51425e..3b7656e34 100644 --- a/packages/@eventual/core-runtime/src/handlers/queue-handler-worker.ts +++ b/packages/@eventual/core-runtime/src/handlers/queue-handler-worker.ts @@ -15,7 +15,7 @@ import { createEventualWorker, type WorkerIntrinsicDeps } from "./worker.js"; export type QueueHandlerDependencies = WorkerIntrinsicDeps; -export interface QueueHandler { +export interface QueueHandlerWorker { ( queueName: string, handlerName: string, @@ -25,7 +25,7 @@ export interface QueueHandler { export function createQueueHandlerWorker( dependencies: QueueHandlerDependencies -): QueueHandler { +): QueueHandlerWorker { return createEventualWorker( { serviceType: ServiceType.QueueHandlerWorker, ...dependencies }, async (queueName, handlerName, items) => { diff --git a/packages/@eventual/core-runtime/src/local/clients/queue-client.ts b/packages/@eventual/core-runtime/src/local/clients/queue-client.ts new file mode 100644 index 000000000..43b5ca98f --- /dev/null +++ b/packages/@eventual/core-runtime/src/local/clients/queue-client.ts @@ -0,0 +1,208 @@ +import { + Schedule, + type DurationSchedule, + type FifoQueue, + type FifoQueueHandlerMessageItem, + type Queue, + type QueueHandlerMessageItem, +} from "@eventual/core"; +import type { QueueSendMessageOperation } from "@eventual/core/internal"; +import { ulid } from "ulidx"; +import { QueueClient } from "../../clients/queue-client.js"; +import { + computeScheduleDate, + type LocalEnvConnector, + type QueueProvider, +} from "../../index.js"; + +export interface QueueRetrieveMessagesRequest { + maxMessages?: number; + visibilityTimeout?: DurationSchedule; +} + +/** + * TODO: implement message deduplication + */ +export class LocalQueueClient extends QueueClient { + constructor( + queueProvider: QueueProvider, + private localConnector: LocalEnvConnector + ) { + super(queueProvider); + } + + private readonly queues: Map = new Map(); + + public receiveMessages( + queueName: string, + request?: QueueRetrieveMessagesRequest + ) { + const queue = this.queues.get(queueName); + return queue ? queue.receiveMessages(request) : []; + } + + public sendMessage(operation: QueueSendMessageOperation): Promise { + let queue = this.queues.get(operation.queueName); + if (!queue) { + queue = new LocalQueue( + this.queueProvider.getQueue(operation.queueName)!, + this.localConnector + ); + this.queues.set(operation.queueName, queue); + } + return queue.sendMessage(operation); + } + + public async changeMessageVisibility( + queueName: string, + receiptHandle: string, + timeout: DurationSchedule + ): Promise { + const queue = this.queues.get(queueName); + return queue?.changeMessageVisibility(receiptHandle, timeout); + } + + public async deleteMessage( + queueName: string, + receiptHandle: string + ): Promise { + const queue = this.queues.get(queueName); + await queue?.deleteMessage(receiptHandle); + } + + public physicalName(queueName: string): string { + return queueName; + } +} + +export class LocalQueue { + constructor( + private queue: FifoQueue | Queue, + private localConnector: LocalEnvConnector + ) {} + + public messages: (FifoQueueHandlerMessageItem | QueueHandlerMessageItem)[] = + []; + + public messageVisibility: Record = {}; + + public receiveMessages(request?: QueueRetrieveMessagesRequest) { + const takeMessages = []; + let i = 0; + const visibilityTimeout = + request?.visibilityTimeout ?? + this.queue.visibilityTimeout ?? + Schedule.duration(30, "seconds"); + + // if we saw a message group ID, track if it was taken or not. + const messageGroupTaken: Record = {}; + + // based on AWS logic, fifo queue can request 10 and non-fifo can request 1000 + const maxMessages = request?.maxMessages ?? this.queue.fifo ? 10 : 1000; + + // when any messages received will be visible + const visibilityTime = computeScheduleDate( + visibilityTimeout, + this.localConnector.getTime() + ); + + while (takeMessages.length < maxMessages && i < this.messages.length) { + const message = this.messages[i++]!; + const messageVisibility = this.messageVisibility[message.receiptHandle]; + + // grab the message group id + const messageGroupId: string | undefined = + (message as FifoQueueHandlerMessageItem).messageGroupId ?? undefined; + + // we'll take from this message group ID if we have not already rejected one. + const messageGroupValid = messageGroupId + ? messageGroupTaken[messageGroupId] ?? true + : true; + + if (messageGroupValid) { + if ( + !messageVisibility || + messageVisibility < this.localConnector.getTime() + ) { + message.receiveCount += 1; + this.messageVisibility[message.id] = visibilityTime; + if (messageGroupId) { + messageGroupTaken[messageGroupId] = true; + } + takeMessages.push(message); + } else { + if (messageGroupId) { + messageGroupTaken[messageGroupId] = false; + } + } + } + } + + if (takeMessages) { + // if any messages were received, set an event to poll against for events. + this.localConnector.scheduleEvent(visibilityTime, { + kind: "QueuePollEvent", + queueName: this.queue.name, + }); + } + + return takeMessages; + } + + public async sendMessage( + operation: QueueSendMessageOperation + ): Promise { + const id = ulid(); + this.messages.push({ + id, + receiptHandle: id, + sequenceNumber: id, + message: operation.message, + messageDeduplicationId: operation.fifo + ? operation.messageDeduplicationId + : (undefined as any), + messageGroupId: operation.fifo + ? operation.messageDeduplicationId + : (undefined as any), + sent: this.localConnector.getTime(), + receiveCount: 0, + }); + + if (operation.delay) { + const visibilityTime = computeScheduleDate( + operation.delay, + operation.message + ); + this.messageVisibility[operation.message.id] = visibilityTime; + // trigger an event to poll this queue for events on the queue when this message is planned to be visible. + this.localConnector.scheduleEvent(visibilityTime, { + kind: "QueuePollEvent", + queueName: this.queue.name, + }); + } else { + // trigger an event to poll this queue for events on the queue. + this.localConnector.pushWorkflowTask({ + kind: "QueuePollEvent", + queueName: this.queue.name, + }); + } + } + + public async changeMessageVisibility( + receiptHandle: string, + timeout: DurationSchedule + ): Promise { + this.messageVisibility[receiptHandle] = computeScheduleDate( + timeout, + this.localConnector.getTime() + ); + } + + public async deleteMessage(receiptHandle: string): Promise { + const indexOf = this.messages.findIndex( + (m) => m.receiptHandle === receiptHandle + ); + this.messages.splice(indexOf, 1); + delete this.messageVisibility[receiptHandle]; + } +} diff --git a/packages/@eventual/core-runtime/src/local/local-container.ts b/packages/@eventual/core-runtime/src/local/local-container.ts index 8771c3e6d..545e134d5 100644 --- a/packages/@eventual/core-runtime/src/local/local-container.ts +++ b/packages/@eventual/core-runtime/src/local/local-container.ts @@ -10,6 +10,7 @@ import { EventClient } from "../clients/event-client.js"; import { ExecutionQueueClient } from "../clients/execution-queue-client.js"; import { LogsClient } from "../clients/logs-client.js"; import { MetricsClient } from "../clients/metrics-client.js"; +import { RuntimeServiceClient } from "../clients/runtime-service-clients.js"; import { TaskClient, TaskWorkerRequest } from "../clients/task-client.js"; import type { TimerClient, TimerRequest } from "../clients/timer-client.js"; import { TransactionClient } from "../clients/transaction-client.js"; @@ -27,6 +28,10 @@ import { createEntityStreamWorker, } from "../handlers/entity-stream-worker.js"; import { Orchestrator, createOrchestrator } from "../handlers/orchestrator.js"; +import { + QueueHandlerWorker, + createQueueHandlerWorker, +} from "../handlers/queue-handler-worker.js"; import { SubscriptionWorker, createSubscriptionWorker, @@ -43,6 +48,10 @@ import { GlobalEntityProvider, } from "../providers/entity-provider.js"; import { InMemoryExecutorProvider } from "../providers/executor-provider.js"; +import { + GlobalQueueProvider, + QueueProvider, +} from "../providers/queue-provider.js"; import { GlobalSubscriptionProvider, SubscriptionProvider, @@ -78,6 +87,7 @@ import { LocalExecutionQueueClient } from "./clients/execution-queue-client.js"; import { LocalLogsClient } from "./clients/logs-client.js"; import { LocalMetricsClient } from "./clients/metrics-client.js"; import { LocalOpenSearchClient } from "./clients/open-search-client.js"; +import { LocalQueueClient } from "./clients/queue-client.js"; import { LocalTaskClient } from "./clients/task-client.js"; import { LocalTimerClient } from "./clients/timer-client.js"; import { LocalTransactionClient } from "./clients/transaction-client.js"; @@ -87,7 +97,6 @@ import { LocalExecutionHistoryStateStore } from "./stores/execution-history-stat import { LocalExecutionHistoryStore } from "./stores/execution-history-store.js"; import { LocalExecutionStore } from "./stores/execution-store.js"; import { LocalTaskStore } from "./stores/task-store.js"; -import { RuntimeServiceClient } from "../clients/runtime-service-clients.js"; export type LocalEvent = | WorkflowTask @@ -95,9 +104,24 @@ export type LocalEvent = | TaskWorkerRequest | LocalEntityStreamEvent | LocalEmittedEvents + | LocalQueuePollEvent | Omit | Omit; +/** + * Event that tells the environment to poll for queue events when one or more exist in the local events. + */ +export interface LocalQueuePollEvent { + kind: "QueuePollEvent"; + queueName: string; +} + +export function isLocalQueuePollEvent( + event: LocalEvent +): event is LocalQueuePollEvent { + return "kind" in event && event.kind === "QueuePollEvent"; +} + export interface LocalEmittedEvents { kind: "EmittedEvents"; events: EventEnvelope[]; @@ -137,16 +161,18 @@ export class LocalContainer { public entityStreamWorker: EntityStreamWorker; public subscriptionWorker: SubscriptionWorker; public transactionWorker: TransactionWorker; + public queueHandlerWorker: QueueHandlerWorker; - public taskClient: TaskClient; public eventClient: EventClient; public executionQueueClient: ExecutionQueueClient; - public workflowClient: WorkflowClient; public logsClient: LogsClient; - public timerClient: TimerClient; public metricsClient: MetricsClient; - public transactionClient: TransactionClient; + public queueClient: LocalQueueClient; public serviceClient: EventualServiceClient; + public taskClient: TaskClient; + public timerClient: TimerClient; + public transactionClient: TransactionClient; + public workflowClient: WorkflowClient; public executionHistoryStateStore: ExecutionHistoryStateStore; public executionHistoryStore: ExecutionHistoryStore; @@ -154,6 +180,7 @@ export class LocalContainer { public taskStore: TaskStore; public entityProvider: EntityProvider; + public queueProvider: QueueProvider; public subscriptionProvider: SubscriptionProvider; public taskProvider: TaskProvider; public workflowProvider: WorkflowProvider; @@ -168,6 +195,7 @@ export class LocalContainer { this.executionStore = new LocalExecutionStore(this.localConnector); this.logsClient = new LocalLogsClient(); this.workflowProvider = new GlobalWorkflowProvider(); + this.queueProvider = new GlobalQueueProvider(); this.workflowClient = new WorkflowClient( this.executionStore, this.logsClient, @@ -206,6 +234,11 @@ export class LocalContainer { this.executionHistoryStateStore = new LocalExecutionHistoryStateStore(); + this.queueClient = new LocalQueueClient( + this.queueProvider, + this.localConnector + ); + this.transactionWorker = createTransactionWorker({ entityStore, entityProvider: this.entityProvider, @@ -232,6 +265,7 @@ export class LocalContainer { bucketStore, entityStore, openSearchClient, + queueClient: this.queueClient, serviceClient: this.serviceClient, serviceSpec: undefined, serviceName: props.serviceName, @@ -243,6 +277,7 @@ export class LocalContainer { bucketStore, entityStore, openSearchClient, + queueClient: this.queueClient, serviceName: props.serviceName, serviceSpec: undefined, serviceUrl: props.serviceUrl, @@ -256,6 +291,7 @@ export class LocalContainer { logAgent, metricsClient: this.metricsClient, openSearchClient, + queueClient: this.queueClient, serviceName: props.serviceName, serviceClient: this.serviceClient, serviceSpec: undefined, @@ -269,6 +305,7 @@ export class LocalContainer { openSearchClient, bucketStore, entityStore, + queueClient: this.queueClient, serviceClient: this.serviceClient, serviceName: props.serviceName, serviceSpec: undefined, @@ -283,11 +320,13 @@ export class LocalContainer { eventClient: this.eventClient, executionQueueClient: this.executionQueueClient, openSearchClient, + queueClient: this.queueClient, taskClient: this.taskClient, timerClient: this.timerClient, transactionClient: this.transactionClient, workflowClient: this.workflowClient, }), + queueClient: this.queueClient, workflowClient: this.workflowClient, timerClient: this.timerClient, serviceName: props.serviceName, @@ -298,6 +337,17 @@ export class LocalContainer { metricsClient: this.metricsClient, }); + this.queueHandlerWorker = createQueueHandlerWorker({ + openSearchClient, + bucketStore, + entityStore, + queueClient: this.queueClient, + serviceClient: this.serviceClient, + serviceName: props.serviceName, + serviceSpec: undefined, + serviceUrl: props.serviceUrl, + }); + /** * Register all of the commands to run. */ @@ -331,6 +381,7 @@ export class LocalContainer { this.commandWorker = createCommandWorker({ entityStore, bucketStore, + queueClient: this.queueClient, serviceClient: this.serviceClient, serviceName: props.serviceName, serviceUrl: props.serviceUrl, diff --git a/packages/@eventual/core-runtime/src/local/local-environment.ts b/packages/@eventual/core-runtime/src/local/local-environment.ts index e177225d1..2b6cf3f28 100644 --- a/packages/@eventual/core-runtime/src/local/local-environment.ts +++ b/packages/@eventual/core-runtime/src/local/local-environment.ts @@ -1,7 +1,7 @@ import { EntityStreamItem, HttpRequest, HttpResponse } from "@eventual/core"; import { - type ServiceSpec, getEventualResources, + type ServiceSpec, } from "@eventual/core/internal"; import { ulid } from "ulidx"; import { isTaskWorkerRequest } from "../clients/task-client.js"; @@ -19,6 +19,7 @@ import { LocalEvent, isLocalEmittedEvents, isLocalEntityStreamEvent, + isLocalQueuePollEvent, } from "./local-container.js"; import { TimeController } from "./time-controller.js"; @@ -121,6 +122,7 @@ export class LocalEnvironment { const entityStreamItems = events.filter(isLocalEntityStreamEvent); const bucketNotificationEvents = events.filter(isBucketNotificationEvent); const localEmittedEvents = events.filter(isLocalEmittedEvents); + const localQueuePollEvents = events.filter(isLocalQueuePollEvent); // run all task requests, don't wait for a result taskWorkerRequests.forEach(async (request) => { @@ -180,6 +182,35 @@ export class LocalEnvironment { // run the orchestrator, but wait for a result. await this.localContainer.orchestrator(workflowTasks); + + // check to see if any queues have messages to process + const queuesToPoll = Array.from( + new Set(localQueuePollEvents.map((s) => s.queueName)) + ); + queuesToPoll.forEach((queueName) => { + const queue = this.localContainer.queueProvider.getQueue(queueName); + const messages = + this.localContainer.queueClient.receiveMessages(queueName); + queue?.handlers.forEach(async (h) => { + const result = await this.localContainer.queueHandlerWorker( + queueName, + h.name, + messages + ); + + const messagesToDelete = result + ? messages.filter((m) => result.failedMessageIds.includes(m.id)) + : messages; + + // when a queue message is handled without error, delete it. + messagesToDelete.forEach((m) => + this.localContainer.queueClient.deleteMessage( + queueName, + m.receiptHandle + ) + ); + }); + }); } } diff --git a/packages/@eventual/core/src/internal/service-spec.ts b/packages/@eventual/core/src/internal/service-spec.ts index 774b75c34..463f5fdb8 100644 --- a/packages/@eventual/core/src/internal/service-spec.ts +++ b/packages/@eventual/core/src/internal/service-spec.ts @@ -325,4 +325,5 @@ export interface QueueSpec { handlers: QueueHandlerSpec[]; fifo: boolean; message?: openapi.SchemaObject; + visibilityTimeout?: DurationSchedule; } diff --git a/packages/@eventual/core/src/queue/fifo.ts b/packages/@eventual/core/src/queue/fifo.ts index 4101fb7ad..cb743c9b2 100644 --- a/packages/@eventual/core/src/queue/fifo.ts +++ b/packages/@eventual/core/src/queue/fifo.ts @@ -174,6 +174,7 @@ export function fifoQueue( handlers, name, fifo: true, + visibilityTimeout: options?.visibilityTimeout, message: options?.message, sendMessage(message, sendOptions) { const messageGroupId = diff --git a/packages/@eventual/core/src/queue/queue.ts b/packages/@eventual/core/src/queue/queue.ts index 4cf430409..b4958f98d 100644 --- a/packages/@eventual/core/src/queue/queue.ts +++ b/packages/@eventual/core/src/queue/queue.ts @@ -127,6 +127,12 @@ export interface Queue export interface QueueOptions { message?: z.Schema; + /** + * The default visibility timeout for messages in the queue. + * + * @default Schedule.duration(30, "seconds") + */ + visibilityTimeout?: DurationSchedule; } export function queue( @@ -143,6 +149,7 @@ export function queue( handlers, name, fifo: false, + visibilityTimeout: options?.visibilityTimeout, message: options?.message, sendMessage(message, options) { return getEventualHook().executeEventualCall( diff --git a/packages/@eventual/testing/src/environment.ts b/packages/@eventual/testing/src/environment.ts index d43fbba2a..b50969562 100644 --- a/packages/@eventual/testing/src/environment.ts +++ b/packages/@eventual/testing/src/environment.ts @@ -28,6 +28,7 @@ import { isBucketNotificationEvent, isLocalEmittedEvents, isLocalEntityStreamEvent, + isLocalQueuePollEvent, isTaskSendEventRequest, isTaskWorkerRequest, isTimerRequest, @@ -379,6 +380,11 @@ export class TestEnvironment extends RuntimeServiceClient { const entityStreamItems = events.filter(isLocalEntityStreamEvent); const bucketNotificationEvents = events.filter(isBucketNotificationEvent); const localEmittedEvents = events.filter(isLocalEmittedEvents); + const localQueuePollEvents = events.filter(isLocalQueuePollEvent); + + const queuesToPoll = Array.from( + new Set(localQueuePollEvents.map((s) => s.queueName)) + ); await Promise.all( // run all task requests, don't wait for a result @@ -442,6 +448,34 @@ export class TestEnvironment extends RuntimeServiceClient { ...localEmittedEvents.map((e) => this.localContainer.subscriptionWorker(e.events) ), + ...queuesToPoll.map(async (queueName) => { + const queue = this.localContainer.queueProvider.getQueue(queueName); + const messages = + this.localContainer.queueClient.receiveMessages(queueName); + await Promise.all( + queue?.handlers.map(async (h) => { + const result = await this.localContainer.queueHandlerWorker( + queueName, + h.name, + messages + ); + + const messagesToDelete = result + ? messages.filter((m) => result.failedMessageIds.includes(m.id)) + : messages; + + // when a queue message is handled without error, delete it. + await Promise.all( + messagesToDelete.map((m) => + this.localContainer.queueClient.deleteMessage( + queueName, + m.receiptHandle + ) + ) + ); + }) ?? [] + ); + }), ] ); } From ef6279271395e1cf134fa523bfc5f46f74083a07 Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Fri, 11 Aug 2023 08:55:41 -0500 Subject: [PATCH 10/20] simplify queue --- packages/@eventual/aws-runtime/src/env.ts | 2 - .../src/handlers/queue-handler-worker.ts | 13 +- .../@eventual/compiler/src/eventual-infer.ts | 17 +- .../src/handlers/queue-handler-worker.ts | 113 +----- .../src/local/local-environment.ts | 36 +- packages/@eventual/core/src/index.ts | 2 +- packages/@eventual/core/src/internal/calls.ts | 5 +- .../@eventual/core/src/internal/resources.ts | 4 +- .../core/src/internal/service-spec.ts | 10 +- packages/@eventual/core/src/queue.ts | 330 ++++++++++++++++ packages/@eventual/core/src/queue/fifo.ts | 363 ------------------ packages/@eventual/core/src/queue/index.ts | 2 - packages/@eventual/core/src/queue/queue.ts | 306 --------------- packages/@eventual/testing/src/environment.ts | 36 +- 14 files changed, 393 insertions(+), 846 deletions(-) create mode 100644 packages/@eventual/core/src/queue.ts delete mode 100644 packages/@eventual/core/src/queue/fifo.ts delete mode 100644 packages/@eventual/core/src/queue/index.ts delete mode 100644 packages/@eventual/core/src/queue/queue.ts diff --git a/packages/@eventual/aws-runtime/src/env.ts b/packages/@eventual/aws-runtime/src/env.ts index bcad9f26b..fea9bc6b3 100644 --- a/packages/@eventual/aws-runtime/src/env.ts +++ b/packages/@eventual/aws-runtime/src/env.ts @@ -24,7 +24,6 @@ export const ENV_NAMES = { ENTITY_NAME: "EVENTUAL_ENTITY_NAME", ENTITY_STREAM_NAME: "EVENTUAL_ENTITY_STREAM_NAME", QUEUE_NAME: "EVENTUAL_QUEUE_NAME", - QUEUE_HANDLER_NAME: "EVENTUAL_QUEUE_HANDLER_NAME", BUCKET_NAME: "EVENTUAL_BUCKET_NAME", BUCKET_HANDLER_NAME: "EVENTUAL_BUCKET_HANDLER_NAME", TRANSACTION_WORKER_ARN: "EVENTUAL_TRANSACTION_WORKER_ARN", @@ -70,7 +69,6 @@ export const entityStreamName = () => tryGetEnv(ENV_NAMES.ENTITY_STREAM_NAME); export const bucketName = () => tryGetEnv(ENV_NAMES.BUCKET_NAME); export const bucketHandlerName = () => tryGetEnv(ENV_NAMES.BUCKET_HANDLER_NAME); export const queueName = () => tryGetEnv(ENV_NAMES.QUEUE_NAME); -export const queueHandlerName = () => tryGetEnv(ENV_NAMES.QUEUE_HANDLER_NAME); export const transactionWorkerArn = () => tryGetEnv(ENV_NAMES.TRANSACTION_WORKER_ARN); export const bucketOverrides = () => { diff --git a/packages/@eventual/aws-runtime/src/handlers/queue-handler-worker.ts b/packages/@eventual/aws-runtime/src/handlers/queue-handler-worker.ts index 3c310e6d3..601cc06c5 100644 --- a/packages/@eventual/aws-runtime/src/handlers/queue-handler-worker.ts +++ b/packages/@eventual/aws-runtime/src/handlers/queue-handler-worker.ts @@ -15,12 +15,7 @@ import { createQueueClient, createServiceClient, } from "../create.js"; -import { - queueHandlerName, - queueName, - serviceName, - serviceUrl, -} from "../env.js"; +import { queueName, serviceName, serviceUrl } from "../env.js"; const worker = createQueueHandlerWorker({ queueClient: createQueueClient(), @@ -48,11 +43,7 @@ export default (async (event) => { sent: new Date(r.attributes.SentTimestamp), } satisfies FifoQueueHandlerMessageItem | QueueHandlerMessageItem) ); - const result = await worker( - getLazy(queueName), - getLazy(queueHandlerName), - items - ); + const result = await worker(getLazy(queueName), items); if (result) { return { batchItemFailures: result.failedMessageIds.map( diff --git a/packages/@eventual/compiler/src/eventual-infer.ts b/packages/@eventual/compiler/src/eventual-infer.ts index 51ea83a13..3837a232e 100644 --- a/packages/@eventual/compiler/src/eventual-infer.ts +++ b/packages/@eventual/compiler/src/eventual-infer.ts @@ -9,7 +9,6 @@ import { getEventualResources, type BucketNotificationHandlerSpec, type CommandSpec, - type QueueHandlerSpec, type QueueSpec, type ServiceSpec, } from "@eventual/core/internal"; @@ -175,19 +174,11 @@ export function inferFromMemory(openApi: ServiceSpec["openApi"]): ServiceSpec { (q) => ({ name: q.name, - handlers: q.handlers.map( - (h) => - ({ - name: h.name, - queueName: q.name, - options: h.options, - batch: h.batch, - fifo: h.fifo, - sourceLocation: h.sourceLocation, - } satisfies QueueHandlerSpec) - ), - message: q.message ? generateSchema(q.message) : undefined, fifo: q.fifo, + handler: { + sourceLocation: q.handler.sourceLocation, + options: q.handler.options, + }, } satisfies QueueSpec) ), }; diff --git a/packages/@eventual/core-runtime/src/handlers/queue-handler-worker.ts b/packages/@eventual/core-runtime/src/handlers/queue-handler-worker.ts index 3b7656e34..cf7aaf6ab 100644 --- a/packages/@eventual/core-runtime/src/handlers/queue-handler-worker.ts +++ b/packages/@eventual/core-runtime/src/handlers/queue-handler-worker.ts @@ -1,16 +1,10 @@ import { - isFifoQueue, - type FifoQueueHandler, type FifoQueueHandlerMessageItem, type QueueHandlerContext, type QueueHandlerMessageItem, } from "@eventual/core"; import { ServiceType, getEventualResource } from "@eventual/core/internal"; -import { - getLazy, - groupedPromiseAllSettled, - promiseAllSettledPartitioned, -} from "../utils.js"; +import { getLazy } from "../utils.js"; import { createEventualWorker, type WorkerIntrinsicDeps } from "./worker.js"; export type QueueHandlerDependencies = WorkerIntrinsicDeps; @@ -18,7 +12,6 @@ export type QueueHandlerDependencies = WorkerIntrinsicDeps; export interface QueueHandlerWorker { ( queueName: string, - handlerName: string, items: (FifoQueueHandlerMessageItem | QueueHandlerMessageItem)[] ): Promise; } @@ -28,95 +21,27 @@ export function createQueueHandlerWorker( ): QueueHandlerWorker { return createEventualWorker( { serviceType: ServiceType.QueueHandlerWorker, ...dependencies }, - async (queueName, handlerName, items) => { + async (queueName, items) => { const queue = getEventualResource("Queue", queueName); if (!queue) throw new Error(`Queue ${queueName} does not exist`); - if (isFifoQueue(queue)) { - const handler = queue.handlers.find((h) => h.name === handlerName); - - if (!handler) throw new Error(`Handler ${handlerName} does not exist`); - - const context: QueueHandlerContext = { - queue: { queueName, fifo: queue.fifo }, - queueHandler: { batch: handler.batch, queueHandlerName: handlerName }, - service: { - serviceName: getLazy(dependencies.serviceName), - serviceUrl: getLazy(dependencies.serviceUrl), - }, - }; - - if (handler.kind === "QueueBatchHandler") { - const result = await handler.handler( - items as FifoQueueHandlerMessageItem[], - context - ); - if (result?.failedMessageIds && result.failedMessageIds.length > 0) { - return { failedMessageIds: result.failedMessageIds }; - } - return undefined; - } else { - // fifo queue needs to maintain order of messages and fail preceding messages. - // in AWS, failing a message out of order will cause an error. - // ex: ([A, B]; A fails and B does not, AWS will throw an error on delete). - const groupResults = await groupedPromiseAllSettled( - items as FifoQueueHandlerMessageItem[], - (item) => item.messageGroupId, - async (item) => { - const result = (handler as FifoQueueHandler).handler( - item, - context - ); - if (result === false) { - throw new Error("Handler reported failure"); - } - return result; - } - ); - - return { - failedMessageIds: Object.values(groupResults).flatMap((g) => - g.rejected.map(([item]) => item.id) - ), - }; - } - } else { - const handler = queue.handlers.find((h) => h.name === handlerName); - - if (!handler) { - throw new Error(`Handler ${handlerName} does not exist`); - } - - const context: QueueHandlerContext = { - queue: { queueName, fifo: queue.fifo }, - queueHandler: { batch: handler.batch, queueHandlerName: handlerName }, - service: { - serviceName: getLazy(dependencies.serviceName), - serviceUrl: getLazy(dependencies.serviceUrl), - }, - }; - - if (handler?.kind === "QueueBatchHandler") { - const result = await handler.handler( - items as QueueHandlerMessageItem[], - context - ); - - if (result?.failedMessageIds && result.failedMessageIds.length > 0) { - return { failedMessageIds: result.failedMessageIds ?? [] }; - } - return undefined; - } else { - // normal queue handler doesn't have a concept of order, pass all messages in any order. - const results = await promiseAllSettledPartitioned( - items, - async (item) => handler.handler(item, context) - ); - - return { - failedMessageIds: results.rejected.map(([item]) => item.id), - }; - } + const handler = queue.handler; + + const context: QueueHandlerContext = { + queue: { queueName, fifo: queue.fifo }, + service: { + serviceName: getLazy(dependencies.serviceName), + serviceUrl: getLazy(dependencies.serviceUrl), + }, + }; + + const result = await handler.handler( + items as FifoQueueHandlerMessageItem[], + context + ); + if (result?.failedMessageIds && result.failedMessageIds.length > 0) { + return { failedMessageIds: result.failedMessageIds }; } + return undefined; } ); } diff --git a/packages/@eventual/core-runtime/src/local/local-environment.ts b/packages/@eventual/core-runtime/src/local/local-environment.ts index 2b6cf3f28..aca5f8122 100644 --- a/packages/@eventual/core-runtime/src/local/local-environment.ts +++ b/packages/@eventual/core-runtime/src/local/local-environment.ts @@ -187,29 +187,25 @@ export class LocalEnvironment { const queuesToPoll = Array.from( new Set(localQueuePollEvents.map((s) => s.queueName)) ); - queuesToPoll.forEach((queueName) => { - const queue = this.localContainer.queueProvider.getQueue(queueName); + queuesToPoll.forEach(async (queueName) => { const messages = this.localContainer.queueClient.receiveMessages(queueName); - queue?.handlers.forEach(async (h) => { - const result = await this.localContainer.queueHandlerWorker( + const result = await this.localContainer.queueHandlerWorker( + queueName, + messages + ); + + const messagesToDelete = result + ? messages.filter((m) => result.failedMessageIds.includes(m.id)) + : messages; + + // when a queue message is handled without error, delete it. + messagesToDelete.forEach((m) => + this.localContainer.queueClient.deleteMessage( queueName, - h.name, - messages - ); - - const messagesToDelete = result - ? messages.filter((m) => result.failedMessageIds.includes(m.id)) - : messages; - - // when a queue message is handled without error, delete it. - messagesToDelete.forEach((m) => - this.localContainer.queueClient.deleteMessage( - queueName, - m.receiptHandle - ) - ); - }); + m.receiptHandle + ) + ); }); } } diff --git a/packages/@eventual/core/src/index.ts b/packages/@eventual/core/src/index.ts index 87439afbe..becb899b5 100644 --- a/packages/@eventual/core/src/index.ts +++ b/packages/@eventual/core/src/index.ts @@ -11,7 +11,7 @@ export * from "./http-method.js"; export * from "./http/index.js"; export * from "./infer.js"; export * from "./logging.js"; -export * from "./queue/index.js"; +export * from "./queue.js"; export * from "./schedule.js"; export * from "./search/index.js"; export * from "./secret.js"; diff --git a/packages/@eventual/core/src/internal/calls.ts b/packages/@eventual/core/src/internal/calls.ts index 1456688c4..2b12c285c 100644 --- a/packages/@eventual/core/src/internal/calls.ts +++ b/packages/@eventual/core/src/internal/calls.ts @@ -7,13 +7,12 @@ import type { } from "../entity/entity.js"; import type { EventEnvelope } from "../event.js"; import type { Execution, ExecutionHandle } from "../execution.js"; -import { FifoContentBasedDeduplication, FifoQueue } from "../queue/fifo.js"; -import type { Queue } from "../queue/queue.js"; import type { DurationSchedule, Schedule } from "../schedule.js"; import type { SearchIndex } from "../search/search-index.js"; import type { Task } from "../task.js"; import type { Workflow, WorkflowExecutionOptions } from "../workflow.js"; import type { SignalTarget } from "./signal.js"; +import { FifoContentBasedDeduplication, FifoQueue, Queue } from "../queue.js"; export type Call = | AwaitTimerCall @@ -233,7 +232,7 @@ export type QueueMethod = Exclude< { [k in keyof Queue]: Queue[k] extends Function ? k : never; }[keyof Queue], - "forEach" | "forEachBatch" | undefined + "handler" | undefined >; export interface QueueCall diff --git a/packages/@eventual/core/src/internal/resources.ts b/packages/@eventual/core/src/internal/resources.ts index dd52fdfce..64c546283 100644 --- a/packages/@eventual/core/src/internal/resources.ts +++ b/packages/@eventual/core/src/internal/resources.ts @@ -2,8 +2,7 @@ import type { Bucket } from "../bucket.js"; import type { Entity } from "../entity/entity.js"; import type { Event } from "../event.js"; import type { AnyCommand } from "../http/command.js"; -import type { FifoQueue } from "../queue/fifo.js"; -import type { Queue } from "../queue/queue.js"; +import { Queue } from "../queue.js"; import type { SearchIndex } from "../search/search-index.js"; import type { Subscription } from "../subscription.js"; import type { Task } from "../task.js"; @@ -20,7 +19,6 @@ type Resource = | Task | Transaction | Queue - | FifoQueue | Workflow; type ResourceKind = Resource["kind"]; diff --git a/packages/@eventual/core/src/internal/service-spec.ts b/packages/@eventual/core/src/internal/service-spec.ts index 463f5fdb8..ee6d136bc 100644 --- a/packages/@eventual/core/src/internal/service-spec.ts +++ b/packages/@eventual/core/src/internal/service-spec.ts @@ -311,19 +311,15 @@ export interface QueueHandlerOptions extends FunctionRuntimeProps { batchingWindow?: DurationSchedule; } -export interface QueueHandlerSpec { - name: Name; - queueName: string; +export interface QueueHandlerSpec { options?: QueueHandlerOptions; - batch: boolean; - fifo: boolean; sourceLocation?: SourceLocation; } export interface QueueSpec { name: Name; - handlers: QueueHandlerSpec[]; + handler: QueueHandlerSpec; fifo: boolean; - message?: openapi.SchemaObject; + contentBasedDeduplication?: boolean; visibilityTimeout?: DurationSchedule; } diff --git a/packages/@eventual/core/src/queue.ts b/packages/@eventual/core/src/queue.ts new file mode 100644 index 000000000..9d64e983e --- /dev/null +++ b/packages/@eventual/core/src/queue.ts @@ -0,0 +1,330 @@ +import { CallKind, createCall, type QueueCall } from "./internal/calls.js"; +import { registerEventualResource } from "./internal/resources.js"; +import type { + QueueHandlerSpec, + QueueSpec, + SourceLocation, +} from "./internal/service-spec.js"; +import type { DurationSchedule } from "./schedule.js"; +import type { ServiceContext } from "./service.js"; + +/** + * Context passed to the handler. + */ +export interface QueueHandlerContext { + /** + * Information about the queue. + */ + queue: { + queueName: string; + fifo: boolean; + }; + /** + * Information about the containing service. + */ + service: ServiceContext; +} + +export interface FifoQueueHandlerMessageItem + extends QueueHandlerMessageItem { + messageGroupId: string; + sequenceNumber: string; + messageDeduplicationId: string; +} + +export interface QueueHandlerMessageItem { + id: string; + receiptHandle: string; + message: Message; + sent: Date; + receiveCount: number; +} + +export interface StandardQueueBatchHandlerFunction< + Message = any, + MessageItem extends QueueHandlerMessageItem = QueueHandlerMessageItem +> { + /** + * Provides the keys, new value + */ + (items: MessageItem[], context: QueueHandlerContext): + | Promise + | void + | { failedMessageIds?: string[] }; +} + +export type QueueBatchHandlerFunction = + | FifoQueueBatchHandlerFunction + | StandardQueueBatchHandlerFunction; + +export type FifoQueueBatchHandlerFunction = + StandardQueueBatchHandlerFunction< + Message, + FifoQueueHandlerMessageItem + >; + +export type QueueBatchHandler> = + QueueHandlerSpec & { + handler: Handler; + }; + +export interface StandardQueueSendMessageOptions { + delay?: DurationSchedule; +} + +export type QueueSendMessageOptions = + | FifoQueueSendMessageOptions + | StandardQueueSendMessageOptions; + +export interface FifoQueueSendMessageOptions + extends StandardQueueSendMessageOptions { + group?: string; + dedupeId?: string; +} + +interface QueueBase + extends Omit, "handler" | "message"> { + kind: "Queue"; + fifo: boolean; + changeMessageVisibility( + receiptHandle: string, + timeout: DurationSchedule + ): Promise; + deleteMessage(receiptHandle: string): Promise; +} + +export interface StandardQueue + extends QueueBase { + fifo: false; + handler: QueueBatchHandler>; + sendMessage( + message: Message, + options?: StandardQueueSendMessageOptions + ): Promise; +} + +export interface FifoQueue + extends QueueBase { + fifo: true; + handler: QueueBatchHandler>; + groupBy?: FifoQueueMessagePropertyReference; + dedupe?: + | FifoQueueMessagePropertyReference + | FifoContentBasedDeduplication; + sendMessage( + message: Message, + options?: FifoQueueSendMessageOptions + ): Promise; +} + +export function isFifoQueue(queue: Queue): queue is FifoQueue { + return queue.fifo; +} + +export type Queue = + | FifoQueue + | StandardQueue; + +export type MessageIdField = { + [K in keyof Message]: K extends string + ? // only include attributes that extend string or number + Message[K] extends string + ? K + : never + : never; +}[keyof Message]; + +export type FifoQueueMessagePropertyReference = + | MessageIdField + | ((message: Message) => string); + +interface QueueOptionsBase { + /** + * The default visibility timeout for messages in the queue. + * + * @default Schedule.duration(30, "seconds") + */ + visibilityTimeout?: DurationSchedule; + fifo?: boolean; + handlerOptions?: QueueHandlerSpec; +} + +export type QueueOptions = + | FifoQueueOptions + | (QueueOptionsBase & { fifo: false }); + +export interface FifoQueueOptions extends QueueOptionsBase { + fifo: true; + groupBy?: FifoQueueMessagePropertyReference; + dedupe?: + | FifoQueueMessagePropertyReference + | FifoContentBasedDeduplication; +} + +export function queue( + ...args: [ + name: Name, + options: QueueOptions, + handler: QueueBatchHandlerFunction + ] +): Queue { + const _args = args as + | [ + name: Name, + options: QueueOptions, + handler: QueueBatchHandlerFunction + ] + | [ + sourceLocation: SourceLocation, + name: Name, + options: QueueOptions, + handler: QueueBatchHandlerFunction + ]; + + const [sourceLocation, name, options, handler] = + _args.length === 4 ? _args : [undefined, ..._args]; + + const { fifo, handlerOptions, visibilityTimeout } = options; + + const queueBase = { + kind: "Queue", + name, + fifo, + visibilityTimeout, + changeMessageVisibility(...args) { + return getEventualHook().executeEventualCall( + createCall(CallKind.QueueCall, { + operation: { + queueName: name, + operation: "changeMessageVisibility", + params: args, + }, + }) + ); + }, + deleteMessage(...args) { + return getEventualHook().executeEventualCall( + createCall(CallKind.QueueCall, { + operation: { + queueName: name, + operation: "deleteMessage", + params: args, + }, + }) + ); + }, + } satisfies Partial>; + + const queue: Queue = fifo + ? >{ + ...queueBase, + handler: { + ...handlerOptions, + sourceLocation, + handler, + }, + fifo, + contentBasedDeduplication: isFifoContentBasedDeduplication( + options.dedupe + ), + groupBy: options.groupBy, + dedupe: options.dedupe, + sendMessage( + message: Message, + sendOptions?: FifoQueueSendMessageOptions + ) { + const messageGroupIdReference = options?.groupBy; + const messageDedupeIdReference = options?.dedupe; + + const messageGroupId = + (sendOptions as FifoQueueSendMessageOptions).group ?? + messageGroupIdReference + ? typeof messageGroupIdReference === "string" + ? message[messageGroupIdReference] + : messageGroupIdReference?.(message) + : undefined; + if (!messageGroupId || typeof messageGroupId !== "string") { + throw new Error( + "Message Group Id must be provided and must be a non-empty string" + ); + } + + const messageDeduplicationId = + (sendOptions as FifoQueueSendMessageOptions).dedupeId ?? + messageDedupeIdReference + ? typeof messageDedupeIdReference === "string" + ? message[messageDedupeIdReference] + : typeof messageDedupeIdReference === "function" + ? messageDedupeIdReference?.(message) + : messageDedupeIdReference + : undefined; + if ( + !messageDeduplicationId || + !( + typeof messageDeduplicationId === "string" || + isFifoContentBasedDeduplication(messageDeduplicationId) + ) + ) { + throw new Error( + "Message Deduplication Id must be provided and a non-empty string or set to { contentBasedDeduplication: true }" + ); + } + + return getEventualHook().executeEventualCall( + createCall(CallKind.QueueCall, { + operation: { + queueName: name, + operation: "sendMessage", + fifo, + messageGroupId, + messageDeduplicationId, + message, + delay: sendOptions?.delay, + }, + }) + ); + }, + } + : >{ + ...queueBase, + handler: { + ...handlerOptions, + sourceLocation, + handler, + }, + fifo, + sendMessage( + message: Message, + sendOptions?: StandardQueueSendMessageOptions + ) { + return getEventualHook().executeEventualCall( + createCall(CallKind.QueueCall, { + operation: { + queueName: name, + operation: "sendMessage", + fifo: false, + delay: sendOptions?.delay, + message, + }, + }) + ); + }, + }; + + return registerEventualResource("Queue", queue as Queue); +} + +/** + * Assertion that content based deduplication is on. + * + * Can be overridden when calling `sendMessage`. + */ +export interface FifoContentBasedDeduplication { + contentBasedDeduplication: true; +} + +export function isFifoContentBasedDeduplication( + value: any +): value is FifoContentBasedDeduplication { + return value && typeof value === "object" && value.contentBasedDeduplication; +} diff --git a/packages/@eventual/core/src/queue/fifo.ts b/packages/@eventual/core/src/queue/fifo.ts deleted file mode 100644 index cb743c9b2..000000000 --- a/packages/@eventual/core/src/queue/fifo.ts +++ /dev/null @@ -1,363 +0,0 @@ -import { CallKind, createCall, type QueueCall } from "../internal/calls.js"; -import { registerEventualResource } from "../internal/resources.js"; -import { - isSourceLocation, - type QueueHandlerOptions, - type SourceLocation, -} from "../internal/service-spec.js"; -import type { - Queue, - QueueBatchHandler, - QueueBatchHandlerFunction, - QueueHandler, - QueueHandlerFunction, - QueueHandlerMessageItem, - QueueOptions, - QueueSendMessageOptions, -} from "./queue.js"; - -export interface FifoQueueHandlerMessageItem - extends QueueHandlerMessageItem { - messageGroupId: string; - sequenceNumber: string; - messageDeduplicationId: string; -} - -/** - * Assertion that content based deduplication is on. - * - * Can be overridden when calling `sendMessage`. - */ -export interface FifoContentBasedDeduplication { - contentBasedDeduplication: true; -} - -export function isFifoContentBasedDeduplication( - value: any -): value is FifoContentBasedDeduplication { - return value && typeof value === "object" && value.contentBasedDeduplication; -} - -export type FifoQueueHandlerFunction = QueueHandlerFunction< - Message, - FifoQueueHandlerMessageItem ->; - -export type FifoQueueBatchHandlerFunction = - QueueBatchHandlerFunction>; - -export interface FifoQueueHandler - extends Omit, "handler" | "fifo"> { - handler: FifoQueueHandlerFunction; - fifo: true; -} - -export interface FifoQueueBatchHandler< - Name extends string = string, - Message = any -> extends Omit, "handler" | "fifo"> { - handler: FifoQueueBatchHandlerFunction; - fifo: true; -} - -export interface FifoQueueSendOptions extends QueueSendMessageOptions { - messageGroupId?: string; - messageDeduplicationId?: string; -} - -export function isFifoQueue(queue: Queue | FifoQueue): queue is FifoQueue { - return queue.fifo; -} - -/** - * TODO: support send and delete batch - */ -export interface FifoQueue - extends Omit< - Queue, - "sendMessage" | "handlers" | "forEach" | "forEachBatch" - > { - handlers: ( - | FifoQueueHandler - | FifoQueueBatchHandler - )[]; - fifo: true; - sendMessage(message: Message, options?: FifoQueueSendOptions): Promise; - forEach( - name: Name, - handler: FifoQueueHandlerFunction - ): FifoQueueHandler; - forEach( - name: Name, - options: QueueHandlerOptions, - handler: FifoQueueHandlerFunction - ): FifoQueueHandler; - forEachBatch( - name: Name, - handler: FifoQueueBatchHandlerFunction - ): FifoQueueBatchHandler; - forEachBatch( - name: Name, - options: QueueHandlerOptions, - handler: FifoQueueBatchHandlerFunction - ): FifoQueueBatchHandler; -} - -export type MessageIdField = { - [K in keyof Message]: K extends string - ? // only include attributes that extend string or number - Message[K] extends string - ? K - : never - : never; -}[keyof Message]; - -export type FifoQueueMessagePropertyReference = - | MessageIdField - | ((message: Message) => string); - -export interface FifoQueueOptions< - Message = any, - MessageGroupId extends - | FifoQueueMessagePropertyReference - | undefined = FifoQueueMessagePropertyReference | undefined, - MessageDeduplicationId extends - | FifoQueueMessagePropertyReference - | undefined - | FifoContentBasedDeduplication = - | FifoQueueMessagePropertyReference - | undefined - | FifoContentBasedDeduplication -> extends QueueOptions { - /** - * The field or function used to compute the `messageGroupId`. - * - * The `messageGroupId` determines how messages are grouped in the Fifo Queue. - * Messages with the same Id will be sent to both forEach and forEachBatch handlers in the order retrieved and cannot progress until - * the first one succeeds. - * - * * Field Name - provide a field to use for the `messageGroupId`. This field must be required and must contain a string value. - * * Getter Function - provide a (synchronous) function to compute the group id from a message on send. - * * undefined - `messageGroupId` will be required on `sendMessage`. - * - * Can be overridden during `sendMessage` by providing a messageGroupId there. - * - * @default undefined - must be set during `sendMessage`. - */ - messageGroupId?: MessageGroupId; - /** - * A field or setting for content deduplication. - * - * * Field Name - provide a field to use for content deduplication. This field must be required and must contain a string value. - * * Getter Function - provide a (synchronous) function to compute the deduplication id from a message on send. - * * `CONTENT_BASED_DEDUPE` - use the content of the message to compute the deduplication id. - * - * Any of these setting can be overridden during send message by providing a messageDeduplicationId there. - */ - messageDeduplicationId?: MessageDeduplicationId; -} - -export function fifoQueue( - name: Name, - options?: FifoQueueOptions -): FifoQueue { - const handlers: ( - | FifoQueueHandler - | FifoQueueBatchHandler - )[] = []; - - const messageGroupIdReference = options?.messageGroupId; - const messageDedupeIdReference = options?.messageDeduplicationId; - - const queue: FifoQueue = { - kind: "Queue", - handlers, - name, - fifo: true, - visibilityTimeout: options?.visibilityTimeout, - message: options?.message, - sendMessage(message, sendOptions) { - const messageGroupId = - sendOptions?.messageGroupId ?? messageGroupIdReference - ? typeof messageGroupIdReference === "string" - ? message[messageGroupIdReference] - : messageGroupIdReference?.(message) - : undefined; - if (!messageGroupId || typeof messageGroupId !== "string") { - throw new Error( - "Message Group Id must be provided and must be a non-empty string" - ); - } - const messageDeduplicationId = - sendOptions?.messageDeduplicationId ?? messageDedupeIdReference - ? typeof messageDedupeIdReference === "string" - ? message[messageDedupeIdReference] - : typeof messageDedupeIdReference === "function" - ? messageDedupeIdReference?.(message) - : messageDedupeIdReference - : undefined; - if ( - !messageDeduplicationId || - !( - typeof messageDeduplicationId === "string" || - isFifoContentBasedDeduplication(messageDeduplicationId) - ) - ) { - throw new Error( - "Message Deduplication Id must be provided and a non-empty string or set to { contentBasedDeduplication: true }" - ); - } - return getEventualHook().executeEventualCall( - createCall(CallKind.QueueCall, { - operation: { - queueName: name, - operation: "sendMessage", - fifo: true, - message, - delay: sendOptions?.delay, - messageGroupId, - messageDeduplicationId, - }, - }) - ); - }, - changeMessageVisibility(...args) { - return getEventualHook().executeEventualCall( - createCall(CallKind.QueueCall, { - operation: { - queueName: name, - operation: "changeMessageVisibility", - params: args, - }, - }) - ); - }, - deleteMessage(...args) { - return getEventualHook().executeEventualCall( - createCall(CallKind.QueueCall, { - operation: { - queueName: name, - operation: "deleteMessage", - params: args, - }, - }) - ); - }, - forEach( - ...args: - | [name: Name, handler: FifoQueueHandlerFunction] - | [ - name: Name, - options: QueueHandlerOptions, - handler: FifoQueueHandlerFunction - ] - | [ - sourceLocation: SourceLocation, - name: Name, - handler: FifoQueueHandlerFunction - ] - | [ - sourceLocation: SourceLocation, - name: Name, - options: QueueHandlerOptions, - handler: FifoQueueHandlerFunction - ] - ) { - const [sourceLocation, handlerName, options, _handler] = - args.length === 4 - ? args - : args.length === 2 - ? [, args[0] as Name, , args[1]] - : isSourceLocation(args[0]) - ? [args[0], args[1] as Name, , args[2]] - : [ - undefined, - ...(args as [ - name: Name, - options: QueueHandlerOptions, - handler: FifoQueueHandlerFunction - ]), - ]; - - if (handlers.some((h) => h.name === handlerName)) { - throw new Error( - `Queue Handler with name ${handlerName} already exists on queue ${name}` - ); - } - - const handler: FifoQueueHandler = { - handler: _handler, - kind: "QueueHandler", - name: handlerName, - sourceLocation, - options, - batch: false, - fifo: true, - queueName: name, - }; - - handlers.push(handler as any); - - return handler; - }, - forEachBatch: ( - ...args: - | [name: Name, handler: FifoQueueBatchHandlerFunction] - | [ - name: Name, - options: QueueHandlerOptions, - handler: FifoQueueBatchHandlerFunction - ] - | [ - sourceLocation: SourceLocation, - name: Name, - handler: FifoQueueBatchHandlerFunction - ] - | [ - sourceLocation: SourceLocation, - name: Name, - options: QueueHandlerOptions, - handler: FifoQueueBatchHandlerFunction - ] - ) => { - const [sourceLocation, handlerName, options, _handler] = - args.length === 4 - ? args - : args.length === 2 - ? [, args[0] as Name, , args[1]] - : isSourceLocation(args[0]) - ? [args[0], args[1] as Name, , args[2]] - : [ - undefined, - ...(args as [ - name: Name, - options: QueueHandlerOptions, - handler: FifoQueueBatchHandlerFunction - ]), - ]; - - if (handlers.some((h) => h.name === handlerName)) { - throw new Error( - `Queue Handler with name ${handlerName} already exists on queue ${name}` - ); - } - - const handler: FifoQueueBatchHandler = { - handler: _handler, - kind: "QueueBatchHandler", - name: handlerName, - sourceLocation, - options, - batch: true, - fifo: true, - queueName: name, - }; - - handlers.push(handler as any); - - return handler; - }, - }; - - return registerEventualResource("Queue", queue); -} diff --git a/packages/@eventual/core/src/queue/index.ts b/packages/@eventual/core/src/queue/index.ts deleted file mode 100644 index 30ecf60a9..000000000 --- a/packages/@eventual/core/src/queue/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./queue.js"; -export * from "./fifo.js"; diff --git a/packages/@eventual/core/src/queue/queue.ts b/packages/@eventual/core/src/queue/queue.ts deleted file mode 100644 index b4958f98d..000000000 --- a/packages/@eventual/core/src/queue/queue.ts +++ /dev/null @@ -1,306 +0,0 @@ -import { z } from "zod"; -import { CallKind, createCall, type QueueCall } from "../internal/calls.js"; -import { registerEventualResource } from "../internal/resources.js"; -import { - isSourceLocation, - type QueueHandlerOptions, - type QueueHandlerSpec, - type QueueSpec, - type SourceLocation, -} from "../internal/service-spec.js"; -import type { DurationSchedule } from "../schedule.js"; -import type { ServiceContext } from "../service.js"; - -export interface QueueHandlerContext { - /** - * Information about the queue. - */ - queue: { - queueName: string; - fifo: boolean; - }; - /** - * Information about the queue handler. - */ - queueHandler: { - queueHandlerName: string; - batch: boolean; - }; - /** - * Information about the containing service. - */ - service: ServiceContext; -} - -export interface QueueHandlerMessageItem { - id: string; - receiptHandle: string; - message: Message; - sent: Date; - receiveCount: number; -} - -export interface QueueHandlerFunction< - Message = any, - MessageItem extends QueueHandlerMessageItem = QueueHandlerMessageItem -> { - /** - * Provides the keys, new value - */ - (item: MessageItem, context: QueueHandlerContext): - | Promise - | void - | false; -} - -export interface QueueBatchHandlerFunction< - Message = any, - MessageItem extends QueueHandlerMessageItem = QueueHandlerMessageItem -> { - /** - * Provides the keys, new value - */ - (items: MessageItem[], context: QueueHandlerContext): - | Promise - | void - | { failedMessageIds?: string[] }; -} - -export interface QueueHandler - extends QueueHandlerSpec { - kind: "QueueHandler"; - handler: QueueHandlerFunction; - sourceLocation?: SourceLocation; - fifo: false; - batch: false; -} - -export interface QueueBatchHandler - extends QueueHandlerSpec { - kind: "QueueBatchHandler"; - handler: QueueBatchHandlerFunction; - sourceLocation?: SourceLocation; - fifo: false; - batch: true; -} - -export interface QueueSendMessageOptions { - delay?: DurationSchedule; -} - -/** - * TODO: support send and delete batch - */ -export interface Queue - extends Omit, "handler" | "message"> { - kind: "Queue"; - handlers: (QueueHandler | QueueBatchHandler)[]; - message?: z.Schema; - sendMessage( - message: Message, - options?: QueueSendMessageOptions - ): Promise; - changeMessageVisibility( - receiptHandle: string, - timeout: DurationSchedule - ): Promise; - deleteMessage(receiptHandle: string): Promise; - forEach( - name: Name, - handler: QueueHandlerFunction - ): QueueHandler; - forEach( - name: Name, - options: QueueHandlerOptions, - handler: QueueHandlerFunction - ): QueueHandler; - forEachBatch( - name: Name, - handler: QueueBatchHandlerFunction - ): QueueBatchHandler; - forEachBatch( - name: Name, - options: QueueHandlerOptions, - handler: QueueBatchHandlerFunction - ): QueueBatchHandler; -} - -export interface QueueOptions { - message?: z.Schema; - /** - * The default visibility timeout for messages in the queue. - * - * @default Schedule.duration(30, "seconds") - */ - visibilityTimeout?: DurationSchedule; -} - -export function queue( - name: Name, - options?: QueueOptions -): Queue { - const handlers: ( - | QueueHandler - | QueueBatchHandler - )[] = []; - - const queue: Queue = { - kind: "Queue", - handlers, - name, - fifo: false, - visibilityTimeout: options?.visibilityTimeout, - message: options?.message, - sendMessage(message, options) { - return getEventualHook().executeEventualCall( - createCall(CallKind.QueueCall, { - operation: { - queueName: name, - operation: "sendMessage", - fifo: false, - message, - delay: options?.delay, - }, - }) - ); - }, - changeMessageVisibility(...args) { - return getEventualHook().executeEventualCall( - createCall(CallKind.QueueCall, { - operation: { - queueName: name, - operation: "changeMessageVisibility", - params: args, - }, - }) - ); - }, - deleteMessage(...args) { - return getEventualHook().executeEventualCall( - createCall(CallKind.QueueCall, { - operation: { - queueName: name, - operation: "deleteMessage", - params: args, - }, - }) - ); - }, - forEach( - ...args: - | [name: Name, handler: QueueHandlerFunction] - | [ - name: Name, - options: QueueHandlerOptions, - handler: QueueHandlerFunction - ] - | [ - sourceLocation: SourceLocation, - name: Name, - handler: QueueHandlerFunction - ] - | [ - sourceLocation: SourceLocation, - name: Name, - options: QueueHandlerOptions, - handler: QueueHandlerFunction - ] - ) { - const [sourceLocation, handlerName, options, _handler] = - args.length === 4 - ? args - : args.length === 2 - ? [, args[0] as Name, , args[1]] - : isSourceLocation(args[0]) - ? [args[0], args[1] as Name, , args[2]] - : [ - undefined, - ...(args as [ - name: Name, - options: QueueHandlerOptions, - handler: QueueHandlerFunction - ]), - ]; - - if (handlers.some((h) => h.name === handlerName)) { - throw new Error( - `Queue Handler with name ${handlerName} already exists on queue ${name}` - ); - } - - const handler: QueueHandler = { - handler: _handler, - kind: "QueueHandler", - name: handlerName, - sourceLocation, - options, - batch: false, - fifo: false, - queueName: name, - }; - - handlers.push(handler as any); - - return handler; - }, - forEachBatch: ( - ...args: - | [name: Name, handler: QueueBatchHandlerFunction] - | [ - name: Name, - options: QueueHandlerOptions, - handler: QueueBatchHandlerFunction - ] - | [ - sourceLocation: SourceLocation, - name: Name, - handler: QueueBatchHandlerFunction - ] - | [ - sourceLocation: SourceLocation, - name: Name, - options: QueueHandlerOptions, - handler: QueueBatchHandlerFunction - ] - ) => { - const [sourceLocation, handlerName, options, _handler] = - args.length === 4 - ? args - : args.length === 2 - ? [, args[0] as Name, , args[1]] - : isSourceLocation(args[0]) - ? [args[0], args[1] as Name, , args[2]] - : [ - undefined, - ...(args as [ - name: Name, - options: QueueHandlerOptions, - handler: QueueBatchHandlerFunction - ]), - ]; - - if (handlers.some((h) => h.name === handlerName)) { - throw new Error( - `Queue Handler with name ${handlerName} already exists on queue ${name}` - ); - } - - const handler: QueueBatchHandler = { - handler: _handler, - kind: "QueueBatchHandler", - name: handlerName, - sourceLocation, - options, - batch: true, - fifo: false, - queueName: name, - }; - - handlers.push(handler as any); - - return handler; - }, - }; - - return registerEventualResource("Queue", queue); -} diff --git a/packages/@eventual/testing/src/environment.ts b/packages/@eventual/testing/src/environment.ts index b50969562..75fdd1845 100644 --- a/packages/@eventual/testing/src/environment.ts +++ b/packages/@eventual/testing/src/environment.ts @@ -449,31 +449,25 @@ export class TestEnvironment extends RuntimeServiceClient { this.localContainer.subscriptionWorker(e.events) ), ...queuesToPoll.map(async (queueName) => { - const queue = this.localContainer.queueProvider.getQueue(queueName); const messages = this.localContainer.queueClient.receiveMessages(queueName); + const result = await this.localContainer.queueHandlerWorker( + queueName, + messages + ); + + const messagesToDelete = result + ? messages.filter((m) => result.failedMessageIds.includes(m.id)) + : messages; + + // when a queue message is handled without error, delete it. await Promise.all( - queue?.handlers.map(async (h) => { - const result = await this.localContainer.queueHandlerWorker( + messagesToDelete.map((m) => + this.localContainer.queueClient.deleteMessage( queueName, - h.name, - messages - ); - - const messagesToDelete = result - ? messages.filter((m) => result.failedMessageIds.includes(m.id)) - : messages; - - // when a queue message is handled without error, delete it. - await Promise.all( - messagesToDelete.map((m) => - this.localContainer.queueClient.deleteMessage( - queueName, - m.receiptHandle - ) - ) - ); - }) ?? [] + m.receiptHandle + ) + ) ); }), ] From 9e744c316b59fe2e1ddf47f6a3689a5c40d21742 Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Fri, 11 Aug 2023 11:03:43 -0500 Subject: [PATCH 11/20] start cdk changes --- packages/@eventual/aws-cdk/src/build.ts | 152 ++++++++---------- .../core-runtime/src/build-manifest.ts | 17 +- 2 files changed, 82 insertions(+), 87 deletions(-) diff --git a/packages/@eventual/aws-cdk/src/build.ts b/packages/@eventual/aws-cdk/src/build.ts index 31dc471cc..03391e774 100644 --- a/packages/@eventual/aws-cdk/src/build.ts +++ b/packages/@eventual/aws-cdk/src/build.ts @@ -1,11 +1,12 @@ import { build, BuildSource, infer } from "@eventual/compiler"; -import { BuildManifest } from "@eventual/core-runtime"; +import { BuildManifest, QueueRuntime } from "@eventual/core-runtime"; import { BucketNotificationHandlerSpec, CommandSpec, EntityStreamSpec, EVENTUAL_SYSTEM_COMMAND_NAMESPACE, - ServiceType, + QueueHandlerSpec, + QueueSpec, SubscriptionSpec, TaskSpec, } from "@eventual/core/internal"; @@ -70,6 +71,17 @@ export interface BuildAWSRuntimeProps { }; } +const WORKER_ENTRY_POINTS = [ + "orchestrator", + "task-worker", + "command-worker", + "subscription-worker", + "entity-stream-worker", + "bucket-handler-worker", + "queue-handler-worker", + "transaction-worker", +] as const; + export async function buildService(request: BuildAWSRuntimeProps) { const outDir = request.outDir; const serviceSpec = await infer(request.entry, request.openApi); @@ -82,16 +94,7 @@ export async function buildService(request: BuildAWSRuntimeProps) { await fs.promises.writeFile(specPath, JSON.stringify(serviceSpec, null, 2)); const [ - [ - // bundle the default handlers first as we refer to them when bundling all of the individual handlers - orchestrator, - monoTaskFunction, - monoCommandFunction, - monoSubscriptionFunction, - monoEntityStreamWorkerFunction, - monoBucketHandlerWorkerFunction, - transactionWorkerFunction, - ], + monolithFunctions, [ // also bundle each of the internal eventual API Functions as they have no dependencies taskFallbackHandler, @@ -119,7 +122,7 @@ export async function buildService(request: BuildAWSRuntimeProps) { subscriptions, commands, commandDefault: { - entry: monoCommandFunction!, + entry: monolithFunctions["command-worker"], spec: { name: "default", }, @@ -142,9 +145,20 @@ export async function buildService(request: BuildAWSRuntimeProps) { })) ), }, + queues: { + queues: await Promise.all( + serviceSpec.queues.map( + async (q) => + ({ + ...q, + handler: await bundleQueueHandler(q), + } satisfies QueueRuntime) + ) + ), + }, system: { entityService: { - transactionWorker: { entry: transactionWorkerFunction! }, + transactionWorker: { entry: monolithFunctions["transaction-worker"] }, }, taskService: { fallbackHandler: { entry: taskFallbackHandler! }, @@ -192,7 +206,7 @@ export async function buildService(request: BuildAWSRuntimeProps) { }, workflowService: { orchestrator: { - entry: orchestrator!, + entry: monolithFunctions.orchestrator!, }, }, }, @@ -207,14 +221,7 @@ export async function buildService(request: BuildAWSRuntimeProps) { return await Promise.all( commandSpecs.map(async (spec) => { return { - entry: await bundleFile( - specPath, - spec, - "command", - "command-worker", - spec.name, - monoCommandFunction! - ), + entry: await bundleFile(spec, "command", "command-worker", spec.name), spec, }; }) @@ -226,12 +233,10 @@ export async function buildService(request: BuildAWSRuntimeProps) { specs.map(async (spec) => { return { entry: await bundleFile( - specPath, spec, "subscription", "subscription-worker", - spec.name, - monoSubscriptionFunction! + spec.name ), spec, }; @@ -243,14 +248,7 @@ export async function buildService(request: BuildAWSRuntimeProps) { return await Promise.all( specs.map(async (spec) => { return { - entry: await bundleFile( - specPath, - spec, - "task", - "task-worker", - spec.name, - monoTaskFunction! - ), + entry: await bundleFile(spec, "task", "task-worker", spec.name), spec, }; }) @@ -262,12 +260,10 @@ export async function buildService(request: BuildAWSRuntimeProps) { specs.map(async (spec) => { return { entry: await bundleFile( - specPath, spec, "entity-streams", "entity-stream-worker", - spec.name, - monoEntityStreamWorkerFunction! + spec.name ), spec, }; @@ -280,12 +276,10 @@ export async function buildService(request: BuildAWSRuntimeProps) { specs.map(async (spec) => { return { entry: await bundleFile( - specPath, spec, "bucket-handlers", "bucket-handler-worker", - spec.name, - monoBucketHandlerWorkerFunction! + spec.name ), spec, }; @@ -293,15 +287,25 @@ export async function buildService(request: BuildAWSRuntimeProps) { ); } + async function bundleQueueHandler(spec: QueueSpec) { + return { + entry: await bundleFile( + spec.handler, + "queue-handlers", + "queue-handler-worker", + spec.name + ), + spec: spec.handler, + }; + } + async function bundleFile< - Spec extends CommandSpec | SubscriptionSpec | TaskSpec + Spec extends CommandSpec | SubscriptionSpec | TaskSpec | QueueHandlerSpec >( - specPath: string, spec: Spec, pathPrefix: string, - entryPoint: string, - name: string, - monoFunction: string + entryPoint: (typeof WORKER_ENTRY_POINTS)[number], + name: string ): Promise { return spec.sourceLocation?.fileName ? // we know the source location of the command, so individually build it from that @@ -315,48 +319,26 @@ export async function buildService(request: BuildAWSRuntimeProps) { injectedEntry: spec.sourceLocation.fileName, injectedServiceSpec: specPath, }) - : monoFunction; + : monolithFunctions[entryPoint]; } - function bundleMonolithDefaultHandlers(specPath: string) { - return Promise.all( - [ - { - name: ServiceType.OrchestratorWorker, - entry: runtimeHandlersEntrypoint("orchestrator"), - }, - { - name: ServiceType.TaskWorker, - entry: runtimeHandlersEntrypoint("task-worker"), - }, - { - name: ServiceType.CommandWorker, - entry: runtimeHandlersEntrypoint("command-worker"), - }, - { - name: ServiceType.Subscription, - entry: runtimeHandlersEntrypoint("subscription-worker"), - }, - { - name: ServiceType.EntityStreamWorker, - entry: runtimeHandlersEntrypoint("entity-stream-worker"), - }, - { - name: ServiceType.BucketNotificationHandlerWorker, - entry: runtimeHandlersEntrypoint("bucket-handler-worker"), - }, - { - name: ServiceType.TransactionWorker, - entry: runtimeHandlersEntrypoint("transaction-worker"), - }, - ] - .map((s) => ({ - ...s, - injectedEntry: request.entry, - injectedServiceSpec: specPath, - })) - .map(buildFunction) - ); + async function bundleMonolithDefaultHandlers(specPath: string) { + return Object.fromEntries( + await Promise.all( + WORKER_ENTRY_POINTS.map( + async (name) => + [ + name, + await buildFunction({ + entry: runtimeHandlersEntrypoint(name), + name, + injectedEntry: request.entry, + injectedServiceSpec: specPath, + }), + ] as const + ) + ) + ) as Record<(typeof WORKER_ENTRY_POINTS)[number], string>; } function bundleEventualSystemFunctions(specPath: string) { diff --git a/packages/@eventual/core-runtime/src/build-manifest.ts b/packages/@eventual/core-runtime/src/build-manifest.ts index 1b5ba460f..307b85507 100644 --- a/packages/@eventual/core-runtime/src/build-manifest.ts +++ b/packages/@eventual/core-runtime/src/build-manifest.ts @@ -1,15 +1,17 @@ import type { - BucketSpec, BucketNotificationHandlerSpec, + BucketSpec, CommandSpec, EntitySpec, EntityStreamSpec, EventSpec, EventualService, + IndexSpec, + QueueHandlerSpec, + QueueSpec, SubscriptionSpec, TaskSpec, TransactionSpec, - IndexSpec, } from "@eventual/core/internal"; export interface BuildManifest { @@ -31,6 +33,7 @@ export interface BuildManifest { commandDefault: CommandFunction; entities: Entities; buckets: Buckets; + queues: Queues; search: Search; system: { entityService: { @@ -64,6 +67,10 @@ export interface BucketRuntime extends Omit { handlers: BucketNotificationHandlerFunction[]; } +export interface QueueRuntime extends Omit { + handler: QueueHandlerFunction; +} + export interface Entities { entities: EntityRuntime[]; transactions: TransactionSpec[]; @@ -73,6 +80,10 @@ export interface Buckets { buckets: BucketRuntime[]; } +export interface Queues { + queues: QueueRuntime[]; +} + export interface Search { indices: IndexSpec[]; } @@ -114,3 +125,5 @@ export type EntityStreamFunction = BundledFunction; export type BucketNotificationHandlerFunction = BundledFunction; + +export type QueueHandlerFunction = BundledFunction; From b261147f80d2ae815e508004205c4b68a084d178 Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Fri, 11 Aug 2023 12:35:11 -0500 Subject: [PATCH 12/20] queue cdk service and shared props --- .../@eventual/aws-cdk/src/queue-service.ts | 182 ++++++++++++++++++ .../@eventual/aws-cdk/src/service-common.ts | 3 + packages/@eventual/aws-cdk/src/service.ts | 36 ++++ .../@eventual/aws-cdk/src/workflow-service.ts | 10 +- .../aws-runtime/src/clients/index.ts | 1 + packages/@eventual/aws-runtime/src/index.ts | 1 + 6 files changed, 231 insertions(+), 2 deletions(-) create mode 100644 packages/@eventual/aws-cdk/src/queue-service.ts create mode 100644 packages/@eventual/aws-runtime/src/clients/index.ts diff --git a/packages/@eventual/aws-cdk/src/queue-service.ts b/packages/@eventual/aws-cdk/src/queue-service.ts new file mode 100644 index 000000000..7d3874166 --- /dev/null +++ b/packages/@eventual/aws-cdk/src/queue-service.ts @@ -0,0 +1,182 @@ +import { + ENV_NAMES, + QueueRuntimeOverrides, + queueServiceQueueSuffix, +} from "@eventual/aws-runtime"; +import type { QueueRuntime } from "@eventual/core-runtime"; +import { IGrantable, IPrincipal, PolicyStatement } from "aws-cdk-lib/aws-iam"; +import type { Function, FunctionProps } from "aws-cdk-lib/aws-lambda"; +import { SqsEventSource } from "aws-cdk-lib/aws-lambda-event-sources"; +import * as sqs from "aws-cdk-lib/aws-sqs"; +import { Duration, Stack } from "aws-cdk-lib/core"; +import { Construct } from "constructs"; +import { EventualResource } from "./resource"; +import { + WorkerServiceConstructProps, + configureWorkerCalls, +} from "./service-common"; +import { ServiceFunction } from "./service-function"; +import { ServiceEntityProps, formatQueueArn, serviceQueueArn } from "./utils"; + +export type QueueHandlerFunctionProps = Omit< + Partial, + "code" | "handler" | "functionName" | "events" +>; + +export type QueueOverrides = Partial< + ServiceEntityProps< + Service, + "Queue", + QueueRuntimeOverrides & + Partial> & { + handler: QueueHandlerFunctionProps; + } + > +>; + +export type ServiceQueues = ServiceEntityProps< + Service, + "Queue", + IQueue +>; + +export interface QueueServiceProps + extends WorkerServiceConstructProps { + queueOverrides?: QueueOverrides; +} + +export class QueueService { + public queues: ServiceQueues; + + constructor(private props: QueueServiceProps) { + const queuesScope = new Construct(props.serviceScope, "Queues"); + + this.queues = Object.fromEntries( + props.build.queues.queues.map((q) => [ + q.name, + new Queue(queuesScope, { + queue: q, + queueService: this, + serviceProps: props, + }), + ]) + ) as ServiceQueues; + } + + public configureSendMessage(func: Function) { + this.addEnvs(func, ENV_NAMES.SERVICE_NAME, ENV_NAMES.QUEUE_OVERRIDES); + this.grantSendAndManageMessage(func); + } + + public grantSendAndManageMessage(grantee: IGrantable) { + // find any queue names that were provided by the service and not computed + const queueNameOverrides = this.props.queueOverrides + ? Object.values( + this.props.queueOverrides as Record + ) + .map((s) => s.queueName) + .filter((s): s is string => !!s) + : []; + + // grants the permission to start any task + grantee.grantPrincipal.addToPrincipalPolicy( + new PolicyStatement({ + actions: [ + "sqs:SendMessage", + "sqs:ChangeMessageVisibility", + "sqs:DeleteMessage", + ], + resources: [ + serviceQueueArn( + this.props.serviceName, + queueServiceQueueSuffix("*"), + false + ), + ...queueNameOverrides.map(formatQueueArn), + ], + }) + ); + } + + private readonly ENV_MAPPINGS = { + [ENV_NAMES.SERVICE_NAME]: () => this.props.serviceName, + [ENV_NAMES.QUEUE_OVERRIDES]: () => + Stack.of(this.props.serviceScope).toJsonString(this.props.queueOverrides), + } as const; + + private addEnvs(func: Function, ...envs: (keyof typeof this.ENV_MAPPINGS)[]) { + envs.forEach((env) => func.addEnvironment(env, this.ENV_MAPPINGS[env]())); + } +} + +interface QueueProps { + serviceProps: QueueServiceProps; + queueService: QueueService; + queue: QueueRuntime; +} + +export interface IQueue { + queue: sqs.Queue; + handler: QueueHandler; +} + +class Queue extends Construct implements IQueue { + public queue: sqs.Queue; + public handler: QueueHandler; + + constructor(scope: Construct, props: QueueProps) { + super(scope, props.queue.name); + + const { handler, ...overrides } = + props.serviceProps.queueOverrides?.[props.queue.name] ?? {}; + + this.queue = new sqs.Queue(this, "Queue", { + ...overrides, + }); + + this.handler = new QueueHandler(this, "Handler", { + queue: this.queue, + queueService: props.queueService, + serviceProps: props.serviceProps, + runtimeQueue: props.queue, + }); + } +} + +interface QueueHandlerProps { + queue: sqs.Queue; + serviceProps: QueueServiceProps; + queueService: QueueService; + runtimeQueue: QueueRuntime; +} + +export class QueueHandler extends Construct implements EventualResource { + public grantPrincipal: IPrincipal; + public handler: Function; + constructor(scope: Construct, id: string, props: QueueHandlerProps) { + super(scope, id); + + const queueName = props.runtimeQueue.name; + + this.handler = new ServiceFunction(this, "Handler", { + build: props.serviceProps.build, + bundledFunction: props.runtimeQueue.handler, + functionNameSuffix: `queue-handler-${queueName}`, + serviceName: props.serviceProps.serviceName, + defaults: { + timeout: Duration.minutes(1), + environment: { + [ENV_NAMES.QUEUE_NAME]: queueName, + ...props.serviceProps.environment, + }, + events: [new SqsEventSource(props.queue, {})], + }, + runtimeProps: props.runtimeQueue.handler.spec.options, + overrides: props.serviceProps.queueOverrides?.[queueName]?.handler, + }); + + configureWorkerCalls(props.serviceProps, this.handler); + + this.grantPrincipal = this.handler.grantPrincipal; + } +} diff --git a/packages/@eventual/aws-cdk/src/service-common.ts b/packages/@eventual/aws-cdk/src/service-common.ts index acea34a9f..babde871c 100644 --- a/packages/@eventual/aws-cdk/src/service-common.ts +++ b/packages/@eventual/aws-cdk/src/service-common.ts @@ -5,6 +5,7 @@ import { BuildOutput } from "./build"; import { CommandService } from "./command-service"; import { EntityService } from "./entity-service"; import { LazyInterface } from "./proxy-construct"; +import { QueueService } from "./queue-service"; import { SearchService } from "./search/search-service"; import { Service } from "./service"; @@ -32,6 +33,7 @@ export interface ServiceConstructProps { * Should match the calls that are supported by the {@link createEventualWorker} function. */ export interface WorkerServiceConstructProps extends ServiceConstructProps { + queueService: LazyInterface>; commandService: LazyInterface>; bucketService: LazyInterface>; entityService: LazyInterface>; @@ -44,6 +46,7 @@ export function configureWorkerCalls( ) { serviceProps.commandService.configureInvokeHttpServiceApi(func); serviceProps.searchService?.configureSearch(func); + serviceProps.queueService.configureSendMessage(func); serviceProps.bucketService.configureReadWriteBuckets(func); serviceProps.entityService.configureReadWriteEntityTable(func); serviceProps.entityService.configureInvokeTransactions(func); diff --git a/packages/@eventual/aws-cdk/src/service.ts b/packages/@eventual/aws-cdk/src/service.ts index a0d7b5229..f67be6549 100644 --- a/packages/@eventual/aws-cdk/src/service.ts +++ b/packages/@eventual/aws-cdk/src/service.ts @@ -56,6 +56,13 @@ import { import { EventService } from "./event-service"; import { grant } from "./grant"; import { lazyInterface } from "./proxy-construct"; +import { + IQueue, + QueueHandler, + QueueOverrides, + QueueService, + ServiceQueues, +} from "./queue-service"; import { EventualResource } from "./resource"; import { SchedulerService } from "./scheduler-service"; import { SearchService, SearchServiceOverrides } from "./search/search-service"; @@ -139,6 +146,10 @@ export interface ServiceProps { * Override the properties of the buckets within the service. */ buckets?: BucketOverrides; + /** + * Override the properties of the queues within the service. + */ + queues?: QueueOverrides; /** * Override the properties of an bucket streams within the service. */ @@ -218,6 +229,10 @@ export class Service extends Construct { * Buckets defined by the service. */ public readonly buckets: ServiceBuckets; + /** + * Queues defined by the service. + */ + public readonly queues: ServiceQueues; /** * Handlers of bucket notification events defined by the service. */ @@ -253,6 +268,7 @@ export class Service extends Construct { public subscriptionsPrincipal: IPrincipal; public entityStreamsPrincipal: IPrincipal; public bucketNotificationHandlersPrincipal: IPrincipal; + public queueHandlersPrincipal: IPrincipal; public readonly system: ServiceSystem; @@ -320,6 +336,7 @@ export class Service extends Construct { const proxyCommandService = lazyInterface>(); const proxyBucketService = lazyInterface>(); const proxyEntityService = lazyInterface>(); + const proxyQueueService = lazyInterface>(); const proxySearchService = lazyInterface>(); const serviceConstructProps: ServiceConstructProps = { @@ -337,6 +354,7 @@ export class Service extends Construct { bucketService: proxyBucketService, commandService: proxyCommandService, entityService: proxyEntityService, + queueService: proxyQueueService, searchService: proxySearchService, }; @@ -443,6 +461,13 @@ export class Service extends Construct { ...workerConstructProps, }); + const queueService = new QueueService({ + ...workerConstructProps, + queueOverrides: props.queues, + }); + proxyQueueService._bind(queueService); + this.queues = queueService.queues; + this.commandService.grantInvokeHttpServiceApi(accessRole); workflowService.grantFilterLogEvents(accessRole); @@ -496,6 +521,13 @@ export class Service extends Construct { ...this.bucketNotificationHandlersList.map((f) => f.grantPrincipal) ) : new UnknownPrincipal({ resource: this }); + this.queueHandlersPrincipal = + this.queueHandlersList.length > 0 || this.local + ? new DeepCompositePrincipal( + ...(this.local ? [this.local.environmentRole] : []), + ...this.queueHandlersList.map((f) => f.grantPrincipal) + ) + : new UnknownPrincipal({ resource: this }); this.grantPrincipal = new DeepCompositePrincipal( this.commandsPrincipal, this.tasksPrincipal, @@ -538,6 +570,10 @@ export class Service extends Construct { return Object.values(this.bucketNotificationHandlers); } + public get queueHandlersList(): QueueHandler[] { + return Object.values(this.queues).map((q) => q.handler); + } + public subscribe( scope: Construct, id: string, diff --git a/packages/@eventual/aws-cdk/src/workflow-service.ts b/packages/@eventual/aws-cdk/src/workflow-service.ts index badcc8ce9..aa5ee4f55 100644 --- a/packages/@eventual/aws-cdk/src/workflow-service.ts +++ b/packages/@eventual/aws-cdk/src/workflow-service.ts @@ -32,10 +32,11 @@ import { EventService } from "./event-service"; import { grant } from "./grant"; import { LazyInterface } from "./proxy-construct"; import { SchedulerService } from "./scheduler-service"; -import type { SearchService } from "./search/search-service"; -import { ServiceConstructProps } from "./service-common"; import { ServiceFunction } from "./service-function"; import type { TaskService } from "./task-service.js"; +import type { SearchService } from "./search/search-service"; +import { ServiceConstructProps } from "./service-common"; +import { QueueService } from "./queue-service"; export interface WorkflowsProps extends ServiceConstructProps { bucketService: LazyInterface>; @@ -45,6 +46,7 @@ export interface WorkflowsProps extends ServiceConstructProps { overrides?: WorkflowServiceOverrides; schedulerService: LazyInterface; taskService: LazyInterface; + queueService: LazyInterface>; } export interface WorkflowServiceOverrides { @@ -482,6 +484,10 @@ export class WorkflowService { * Bucket Call */ this.props.bucketService.configureReadWriteBuckets(this.orchestrator); + /** + * Queue Calls + */ + this.props.queueService.configureSendMessage(this.orchestrator); } private readonly ENV_MAPPINGS = { diff --git a/packages/@eventual/aws-runtime/src/clients/index.ts b/packages/@eventual/aws-runtime/src/clients/index.ts new file mode 100644 index 000000000..b9ae07f5a --- /dev/null +++ b/packages/@eventual/aws-runtime/src/clients/index.ts @@ -0,0 +1 @@ +export * from "./queue-client.js"; diff --git a/packages/@eventual/aws-runtime/src/index.ts b/packages/@eventual/aws-runtime/src/index.ts index 981deee20..36134fd52 100644 --- a/packages/@eventual/aws-runtime/src/index.ts +++ b/packages/@eventual/aws-runtime/src/index.ts @@ -1,3 +1,4 @@ +export * from "./clients/index.js"; export * from "./env.js"; export * from "./stores/index.js"; export * from "./utils.js"; From 7c94130ca8f5a4f4d6dc2fce7a631204b0513738 Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Mon, 14 Aug 2023 06:55:30 -0600 Subject: [PATCH 13/20] runtime test --- apps/tests/aws-runtime/test/test-service.ts | 70 +++++++++++++++++ apps/tests/aws-runtime/test/tester.test.ts | 3 + .../src/handlers/queue-handler-worker.ts | 32 ++++---- .../src/handlers/queue-handler-worker.ts | 4 +- .../src/local/clients/queue-client.ts | 8 +- packages/@eventual/core/src/queue.ts | 75 ++++++++++++------- 6 files changed, 146 insertions(+), 46 deletions(-) diff --git a/apps/tests/aws-runtime/test/test-service.ts b/apps/tests/aws-runtime/test/test-service.ts index d7b2a24a9..a458ce919 100644 --- a/apps/tests/aws-runtime/test/test-service.ts +++ b/apps/tests/aws-runtime/test/test-service.ts @@ -24,6 +24,7 @@ import { event, expectSignal, index, + queue, sendSignal, sendTaskHeartbeat, signal, @@ -1066,6 +1067,75 @@ export const bucketWorkflow = workflow( } ); +const queueSignal = signal<{ n: number }>("queueSignal"); +const fifoQueueSignal = signal<{ n: number }>("fifoQueueSignal"); + +export const testQueue = queue<{ + executionId: string; + source: "workflow" | "queue"; + n: number; +}>("testQueue", {}, async (messages) => { + await Promise.all( + messages.map(async (m) => { + if (m.message.source === "workflow") { + await testFifoQueue.sendMessage({ + executionId: m.message.executionId, + source: "queue", + n: m.message.n + 1, + }); + } else if (m.message.source === "queue") { + await queueSignal.sendSignal(m.message.executionId, { + n: m.message.n + 1, + }); + } + }) + ); +}); + +export const testFifoQueue = queue<{ + executionId: string; + source: "workflow" | "queue"; + n: number; +}>( + "testFifoQueue", + { + fifo: true, + groupBy: (m) => m.executionId, + dedupe: { contentBasedDeduplication: true }, + }, + async (messages) => { + await Promise.all( + messages.map(async (m) => { + if (m.message.source === "workflow") { + await testQueue.sendMessage({ + executionId: m.message.executionId, + source: "queue", + n: m.message.n + 1, + }); + } else if (m.message.source === "queue") { + await fifoQueueSignal.sendSignal(m.message.executionId, { + n: m.message.n + 1, + }); + } + }) + ); + } +); + +export const queueWorkflow = workflow( + "queueWorkflow", + async (_, { execution: { id } }) => { + const [, , { n }, { n: fifo_n }] = await Promise.all([ + testQueue.sendMessage({ executionId: id, source: "workflow", n: 1 }), + testFifoQueue.sendMessage({ executionId: id, source: "workflow", n: 1 }), + queueSignal.expectSignal(), + fifoQueueSignal.expectSignal(), + ]); + + return { n, fifo_n }; + } +); + export const hello3 = api.post("/hello3", () => { return new HttpResponse("hello?"); }); diff --git a/apps/tests/aws-runtime/test/tester.test.ts b/apps/tests/aws-runtime/test/tester.test.ts index fba2fd287..d1bcf83e8 100644 --- a/apps/tests/aws-runtime/test/tester.test.ts +++ b/apps/tests/aws-runtime/test/tester.test.ts @@ -20,6 +20,7 @@ import { failedWorkflow, heartbeatWorkflow, parentWorkflow, + queueWorkflow, timedOutWorkflow, timedWorkflow, transactionWorkflow, @@ -251,6 +252,8 @@ eventualRuntimeTestHarness( signalResult4: { data: "hello again again again!" }, copied: "hello again again again!", }); + + testCompletion("queue", queueWorkflow, { n: 3, fifo_n: 3 }); }, { name: "s3 persist failures", diff --git a/packages/@eventual/aws-runtime/src/handlers/queue-handler-worker.ts b/packages/@eventual/aws-runtime/src/handlers/queue-handler-worker.ts index 601cc06c5..b6e141f94 100644 --- a/packages/@eventual/aws-runtime/src/handlers/queue-handler-worker.ts +++ b/packages/@eventual/aws-runtime/src/handlers/queue-handler-worker.ts @@ -4,7 +4,7 @@ import "@eventual/injected/entry"; import type { FifoQueueHandlerMessageItem, - QueueHandlerMessageItem, + StandardQueueHandlerMessageItem, } from "@eventual/core"; import { createQueueHandlerWorker, getLazy } from "@eventual/core-runtime"; import type { SQSBatchItemFailure, SQSHandler } from "aws-lambda"; @@ -29,20 +29,22 @@ const worker = createQueueHandlerWorker({ }); export default (async (event) => { - const items: (FifoQueueHandlerMessageItem | QueueHandlerMessageItem)[] = - event.Records.map( - (r) => - ({ - id: r.messageId, - message: r.body, - sequenceNumber: r.attributes.SequenceNumber, - messageDeduplicationId: r.attributes.MessageDeduplicationId, - messageGroupId: r.attributes.MessageGroupId, - receiptHandle: r.receiptHandle, - receiveCount: Number(r.attributes.ApproximateReceiveCount), - sent: new Date(r.attributes.SentTimestamp), - } satisfies FifoQueueHandlerMessageItem | QueueHandlerMessageItem) - ); + const items: ( + | FifoQueueHandlerMessageItem + | StandardQueueHandlerMessageItem + )[] = event.Records.map( + (r) => + ({ + id: r.messageId, + message: r.body, + sequenceNumber: r.attributes.SequenceNumber, + messageDeduplicationId: r.attributes.MessageDeduplicationId, + messageGroupId: r.attributes.MessageGroupId, + receiptHandle: r.receiptHandle, + receiveCount: Number(r.attributes.ApproximateReceiveCount), + sent: new Date(r.attributes.SentTimestamp), + } satisfies FifoQueueHandlerMessageItem | StandardQueueHandlerMessageItem) + ); const result = await worker(getLazy(queueName), items); if (result) { return { diff --git a/packages/@eventual/core-runtime/src/handlers/queue-handler-worker.ts b/packages/@eventual/core-runtime/src/handlers/queue-handler-worker.ts index cf7aaf6ab..e52055b43 100644 --- a/packages/@eventual/core-runtime/src/handlers/queue-handler-worker.ts +++ b/packages/@eventual/core-runtime/src/handlers/queue-handler-worker.ts @@ -1,7 +1,7 @@ import { type FifoQueueHandlerMessageItem, type QueueHandlerContext, - type QueueHandlerMessageItem, + type StandardQueueHandlerMessageItem, } from "@eventual/core"; import { ServiceType, getEventualResource } from "@eventual/core/internal"; import { getLazy } from "../utils.js"; @@ -12,7 +12,7 @@ export type QueueHandlerDependencies = WorkerIntrinsicDeps; export interface QueueHandlerWorker { ( queueName: string, - items: (FifoQueueHandlerMessageItem | QueueHandlerMessageItem)[] + items: (FifoQueueHandlerMessageItem | StandardQueueHandlerMessageItem)[] ): Promise; } diff --git a/packages/@eventual/core-runtime/src/local/clients/queue-client.ts b/packages/@eventual/core-runtime/src/local/clients/queue-client.ts index 43b5ca98f..3720acf47 100644 --- a/packages/@eventual/core-runtime/src/local/clients/queue-client.ts +++ b/packages/@eventual/core-runtime/src/local/clients/queue-client.ts @@ -4,7 +4,7 @@ import { type FifoQueue, type FifoQueueHandlerMessageItem, type Queue, - type QueueHandlerMessageItem, + type StandardQueueHandlerMessageItem, } from "@eventual/core"; import type { QueueSendMessageOperation } from "@eventual/core/internal"; import { ulid } from "ulidx"; @@ -81,8 +81,10 @@ export class LocalQueue { private localConnector: LocalEnvConnector ) {} - public messages: (FifoQueueHandlerMessageItem | QueueHandlerMessageItem)[] = - []; + public messages: ( + | FifoQueueHandlerMessageItem + | StandardQueueHandlerMessageItem + )[] = []; public messageVisibility: Record = {}; diff --git a/packages/@eventual/core/src/queue.ts b/packages/@eventual/core/src/queue.ts index 9d64e983e..8a5aee13f 100644 --- a/packages/@eventual/core/src/queue.ts +++ b/packages/@eventual/core/src/queue.ts @@ -26,13 +26,13 @@ export interface QueueHandlerContext { } export interface FifoQueueHandlerMessageItem - extends QueueHandlerMessageItem { + extends StandardQueueHandlerMessageItem { messageGroupId: string; sequenceNumber: string; messageDeduplicationId: string; } -export interface QueueHandlerMessageItem { +export interface StandardQueueHandlerMessageItem { id: string; receiptHandle: string; message: Message; @@ -40,9 +40,9 @@ export interface QueueHandlerMessageItem { receiveCount: number; } -export interface StandardQueueBatchHandlerFunction< +export interface QueueBatchHandlerFunctionBase< Message = any, - MessageItem extends QueueHandlerMessageItem = QueueHandlerMessageItem + MessageItem extends StandardQueueHandlerMessageItem = StandardQueueHandlerMessageItem > { /** * Provides the keys, new value @@ -53,20 +53,27 @@ export interface StandardQueueBatchHandlerFunction< | { failedMessageIds?: string[] }; } -export type QueueBatchHandlerFunction = - | FifoQueueBatchHandlerFunction - | StandardQueueBatchHandlerFunction; +export type QueueBatchHandlerFunction< + Message, + Fifo extends boolean | undefined +> = Fifo extends true + ? FifoQueueBatchHandlerFunction + : StandardQueueBatchHandlerFunction; -export type FifoQueueBatchHandlerFunction = - StandardQueueBatchHandlerFunction< +export type StandardQueueBatchHandlerFunction = + QueueBatchHandlerFunctionBase< Message, - FifoQueueHandlerMessageItem + StandardQueueHandlerMessageItem >; -export type QueueBatchHandler> = - QueueHandlerSpec & { - handler: Handler; - }; +export type FifoQueueBatchHandlerFunction = + QueueBatchHandlerFunctionBase>; + +export type QueueBatchHandler< + Handler extends QueueBatchHandlerFunction +> = QueueHandlerSpec & { + handler: Handler; +}; export interface StandardQueueSendMessageOptions { delay?: DurationSchedule; @@ -138,22 +145,22 @@ export type FifoQueueMessagePropertyReference = | MessageIdField | ((message: Message) => string); -interface QueueOptionsBase { - /** +interface QueueOptionsBase { + /** k * The default visibility timeout for messages in the queue. * * @default Schedule.duration(30, "seconds") */ visibilityTimeout?: DurationSchedule; - fifo?: boolean; + fifo?: Fifo; handlerOptions?: QueueHandlerSpec; } export type QueueOptions = | FifoQueueOptions - | (QueueOptionsBase & { fifo: false }); + | QueueOptionsBase; -export interface FifoQueueOptions extends QueueOptionsBase { +export interface FifoQueueOptions extends QueueOptionsBase { fifo: true; groupBy?: FifoQueueMessagePropertyReference; dedupe?: @@ -162,23 +169,41 @@ export interface FifoQueueOptions extends QueueOptionsBase { } export function queue( + ...args: [ + name: Name, + options: FifoQueueOptions, + handler: FifoQueueBatchHandlerFunction + ] +): FifoQueue; +export function queue( + ...args: [ + name: Name, + options: QueueOptions, + handler: StandardQueueBatchHandlerFunction + ] +): StandardQueue; +export function queue< + Message, + Name extends string = string, + Fifo extends boolean | undefined = undefined +>( ...args: [ name: Name, options: QueueOptions, - handler: QueueBatchHandlerFunction + handler: QueueBatchHandlerFunction ] ): Queue { const _args = args as | [ name: Name, options: QueueOptions, - handler: QueueBatchHandlerFunction + handler: QueueBatchHandlerFunction ] | [ sourceLocation: SourceLocation, name: Name, options: QueueOptions, - handler: QueueBatchHandlerFunction + handler: QueueBatchHandlerFunction ]; const [sourceLocation, name, options, handler] = @@ -237,8 +262,7 @@ export function queue( const messageDedupeIdReference = options?.dedupe; const messageGroupId = - (sendOptions as FifoQueueSendMessageOptions).group ?? - messageGroupIdReference + sendOptions?.group ?? messageGroupIdReference ? typeof messageGroupIdReference === "string" ? message[messageGroupIdReference] : messageGroupIdReference?.(message) @@ -250,8 +274,7 @@ export function queue( } const messageDeduplicationId = - (sendOptions as FifoQueueSendMessageOptions).dedupeId ?? - messageDedupeIdReference + sendOptions?.dedupeId ?? messageDedupeIdReference ? typeof messageDedupeIdReference === "string" ? message[messageDedupeIdReference] : typeof messageDedupeIdReference === "function" From b43fdc66822c35f3adb2b17e53a6b544affc2ed7 Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Wed, 23 Aug 2023 08:45:43 -0500 Subject: [PATCH 14/20] tests pass --- .../@eventual/aws-cdk/src/queue-service.ts | 32 ++++++++++++++++--- .../@eventual/aws-cdk/src/service-function.ts | 3 +- packages/@eventual/aws-cdk/src/utils.ts | 4 +-- .../aws-runtime/src/clients/queue-client.ts | 9 ++++-- packages/@eventual/aws-runtime/src/env.ts | 3 +- .../src/handlers/queue-handler-worker.ts | 2 +- packages/@eventual/aws-runtime/src/utils.ts | 32 +++++++++++++++++-- .../@eventual/compiler/src/eventual-infer.ts | 3 ++ .../src/local/clients/queue-client.ts | 4 +-- .../core/src/internal/service-spec.ts | 1 + packages/@eventual/core/src/queue.ts | 15 +++++++-- 11 files changed, 89 insertions(+), 19 deletions(-) diff --git a/packages/@eventual/aws-cdk/src/queue-service.ts b/packages/@eventual/aws-cdk/src/queue-service.ts index 7d3874166..ff6e65d99 100644 --- a/packages/@eventual/aws-cdk/src/queue-service.ts +++ b/packages/@eventual/aws-cdk/src/queue-service.ts @@ -1,9 +1,11 @@ import { ENV_NAMES, QueueRuntimeOverrides, + queueServiceQueueName, queueServiceQueueSuffix, } from "@eventual/aws-runtime"; -import type { QueueRuntime } from "@eventual/core-runtime"; +import { DEFAULT_QUEUE_VISIBILITY_TIMEOUT } from "@eventual/core"; +import { QueueRuntime, computeDurationSeconds } from "@eventual/core-runtime"; import { IGrantable, IPrincipal, PolicyStatement } from "aws-cdk-lib/aws-iam"; import type { Function, FunctionProps } from "aws-cdk-lib/aws-lambda"; import { SqsEventSource } from "aws-cdk-lib/aws-lambda-event-sources"; @@ -78,7 +80,7 @@ export class QueueService { .filter((s): s is string => !!s) : []; - // grants the permission to start any task + // grants the permission to interact with any queue grantee.grantPrincipal.addToPrincipalPolicy( new PolicyStatement({ actions: [ @@ -92,7 +94,7 @@ export class QueueService { queueServiceQueueSuffix("*"), false ), - ...queueNameOverrides.map(formatQueueArn), + ...queueNameOverrides.map((n) => formatQueueArn(n)), ], }) ); @@ -131,6 +133,21 @@ class Queue extends Construct implements IQueue { props.serviceProps.queueOverrides?.[props.queue.name] ?? {}; this.queue = new sqs.Queue(this, "Queue", { + contentBasedDeduplication: props.queue.contentBasedDeduplication, + deliveryDelay: props.queue.delay + ? Duration.seconds(computeDurationSeconds(props.queue.delay)) + : undefined, + fifo: props.queue.fifo, + queueName: queueServiceQueueName( + props.serviceProps.serviceName, + props.queue.name, + props.queue.fifo + ), + visibilityTimeout: props.queue.visibilityTimeout + ? Duration.seconds( + computeDurationSeconds(props.queue.visibilityTimeout) + ) + : undefined, ...overrides, }); @@ -164,7 +181,14 @@ export class QueueHandler extends Construct implements EventualResource { functionNameSuffix: `queue-handler-${queueName}`, serviceName: props.serviceProps.serviceName, defaults: { - timeout: Duration.minutes(1), + // CFN enforces that the queue's visibility timeout is greater than or equal to the lambda timeout. + // default queue visibility is 30 seconds. + timeout: Duration.seconds( + computeDurationSeconds( + props.runtimeQueue.visibilityTimeout ?? + DEFAULT_QUEUE_VISIBILITY_TIMEOUT + ) + ), environment: { [ENV_NAMES.QUEUE_NAME]: queueName, ...props.serviceProps.environment, diff --git a/packages/@eventual/aws-cdk/src/service-function.ts b/packages/@eventual/aws-cdk/src/service-function.ts index b5208696b..63376df44 100644 --- a/packages/@eventual/aws-cdk/src/service-function.ts +++ b/packages/@eventual/aws-cdk/src/service-function.ts @@ -4,7 +4,7 @@ import { BundledFunction, computeDurationSeconds, } from "@eventual/core-runtime"; -import { Duration } from "aws-cdk-lib/core"; +import { Duration, Stack } from "aws-cdk-lib/core"; import { Function, FunctionProps } from "aws-cdk-lib/aws-lambda"; import { Construct } from "constructs"; import type { BuildOutput } from "./build"; @@ -56,6 +56,7 @@ export class ServiceFunction extends Function { environment: { NODE_OPTIONS: "--enable-source-maps", [ENV_NAMES.SERVICE_NAME]: props.build.serviceName, + [ENV_NAMES.AWS_ACCOUNT_ID]: Stack.of(scope).account, ...baseFnProps.environment, ...props.defaults?.environment, ...props.overrides?.environment, diff --git a/packages/@eventual/aws-cdk/src/utils.ts b/packages/@eventual/aws-cdk/src/utils.ts index 8f565bad5..15e5cd72a 100644 --- a/packages/@eventual/aws-cdk/src/utils.ts +++ b/packages/@eventual/aws-cdk/src/utils.ts @@ -126,6 +126,6 @@ export function serviceQueueArn( ); } -export function formatQueueArn(queueName: string) { - return `arn:aws:sqs:::${queueName}`; +export function formatQueueArn(queueName: string, region = "*", account = "*") { + return `arn:aws:sqs:${region}:${account}:${queueName}`; } diff --git a/packages/@eventual/aws-runtime/src/clients/queue-client.ts b/packages/@eventual/aws-runtime/src/clients/queue-client.ts index 7fc61b8f0..f7e0be159 100644 --- a/packages/@eventual/aws-runtime/src/clients/queue-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/queue-client.ts @@ -44,7 +44,7 @@ export class AWSQueueClient extends QueueClient { ): Promise { await this.props.sqs.send( new SendMessageCommand({ - MessageBody: operation.message, + MessageBody: JSON.stringify(operation.message), QueueUrl: this.physicalQueueUrl(operation.queueName), DelaySeconds: operation.delay ? computeDurationSeconds(operation.delay) @@ -98,9 +98,14 @@ export class AWSQueueClient extends QueueClient { public override physicalName(queueName: string) { const overrides = getLazy(this.props.queueOverrides); const nameOverride = overrides[queueName]?.queueName; + const queue = this.getQueue(queueName); return ( nameOverride ?? - queueServiceQueueName(getLazy(this.props.serviceName), queueName) + queueServiceQueueName( + getLazy(this.props.serviceName), + queueName, + queue.fifo + ) ); } } diff --git a/packages/@eventual/aws-runtime/src/env.ts b/packages/@eventual/aws-runtime/src/env.ts index fea9bc6b3..54305f83e 100644 --- a/packages/@eventual/aws-runtime/src/env.ts +++ b/packages/@eventual/aws-runtime/src/env.ts @@ -4,6 +4,7 @@ import { BucketRuntimeOverrides } from "./stores/bucket-store.js"; import { QueueRuntimeOverrides } from "./clients/queue-client.js"; export const ENV_NAMES = { + AWS_ACCOUNT_ID: "EVENTUAL_AWS_ACCOUNT_ID", SERVICE_NAME: "EVENTUAL_SERVICE_NAME", SERVICE_URL: "EVENTUAL_SERVICE_URL", EXECUTION_TABLE_NAME: "EVENTUAL_EXECUTION_TABLE_NAME", @@ -38,7 +39,7 @@ export function tryGetEnv(name: string) { ) as T; } -export const awsAccount = () => tryGetEnv("AWS_ACCOUNT_ID"); +export const awsAccount = () => tryGetEnv(ENV_NAMES.AWS_ACCOUNT_ID); export const awsRegion = () => tryGetEnv("AWS_REGION"); export const serviceName = () => tryGetEnv(ENV_NAMES.SERVICE_NAME); export const openSearchEndpoint = () => diff --git a/packages/@eventual/aws-runtime/src/handlers/queue-handler-worker.ts b/packages/@eventual/aws-runtime/src/handlers/queue-handler-worker.ts index b6e141f94..b80e81143 100644 --- a/packages/@eventual/aws-runtime/src/handlers/queue-handler-worker.ts +++ b/packages/@eventual/aws-runtime/src/handlers/queue-handler-worker.ts @@ -36,7 +36,7 @@ export default (async (event) => { (r) => ({ id: r.messageId, - message: r.body, + message: JSON.parse(r.body), sequenceNumber: r.attributes.SequenceNumber, messageDeduplicationId: r.attributes.MessageDeduplicationId, messageGroupId: r.attributes.MessageGroupId, diff --git a/packages/@eventual/aws-runtime/src/utils.ts b/packages/@eventual/aws-runtime/src/utils.ts index 1b322fa3d..b678a07bd 100644 --- a/packages/@eventual/aws-runtime/src/utils.ts +++ b/packages/@eventual/aws-runtime/src/utils.ts @@ -222,6 +222,27 @@ export function serviceFunctionName(serviceName: string, suffix: string) { ); } +const FIFO_SUFFIX = ".fifo"; + +/** + * Creates a queue name with a max length of 80 characters + * + * The name will be the whole service name followed by the suffix trimmed to fit 80 characters with .fifo appended on the end. + */ +export function serviceQueueName( + serviceName: string, + suffix: string, + fifo: boolean +) { + const serviceNameAndSeparatorLength = serviceName.length + 1; + const remaining = + 80 - serviceNameAndSeparatorLength - (fifo ? FIFO_SUFFIX.length : 0); + // the . in .fifo is only valid as part of the .fifo suffix + return `${sanitizeFunctionName( + `${serviceName}-${suffix.substring(0, remaining)}` + )}${fifo ? FIFO_SUFFIX : ""}`; +} + /** * Bucket names must: * * be between 3 and 63 characters long (inc) @@ -291,13 +312,18 @@ export function bucketServiceBucketName( */ export function queueServiceQueueName( serviceName: string, - queueName: string + queueName: string, + fifo: boolean ): string { - return serviceFunctionName(serviceName, queueServiceQueueSuffix(queueName)); + return serviceQueueName( + serviceName, + queueServiceQueueSuffix(queueName), + fifo + ); } /** - * Valid lambda function names contains letters, numbers, dash, or underscore and no spaces. + * Valid lambda function and sqs queue names contains letters, numbers, dash, or underscore and no spaces. */ export function sanitizeFunctionName(name: string) { return ( diff --git a/packages/@eventual/compiler/src/eventual-infer.ts b/packages/@eventual/compiler/src/eventual-infer.ts index 3837a232e..b64107c95 100644 --- a/packages/@eventual/compiler/src/eventual-infer.ts +++ b/packages/@eventual/compiler/src/eventual-infer.ts @@ -179,6 +179,9 @@ export function inferFromMemory(openApi: ServiceSpec["openApi"]): ServiceSpec { sourceLocation: q.handler.sourceLocation, options: q.handler.options, }, + contentBasedDeduplication: q.contentBasedDeduplication, + delay: q.delay, + visibilityTimeout: q.visibilityTimeout, } satisfies QueueSpec) ), }; diff --git a/packages/@eventual/core-runtime/src/local/clients/queue-client.ts b/packages/@eventual/core-runtime/src/local/clients/queue-client.ts index 3720acf47..963bb1b10 100644 --- a/packages/@eventual/core-runtime/src/local/clients/queue-client.ts +++ b/packages/@eventual/core-runtime/src/local/clients/queue-client.ts @@ -1,5 +1,5 @@ import { - Schedule, + DEFAULT_QUEUE_VISIBILITY_TIMEOUT, type DurationSchedule, type FifoQueue, type FifoQueueHandlerMessageItem, @@ -94,7 +94,7 @@ export class LocalQueue { const visibilityTimeout = request?.visibilityTimeout ?? this.queue.visibilityTimeout ?? - Schedule.duration(30, "seconds"); + DEFAULT_QUEUE_VISIBILITY_TIMEOUT; // if we saw a message group ID, track if it was taken or not. const messageGroupTaken: Record = {}; diff --git a/packages/@eventual/core/src/internal/service-spec.ts b/packages/@eventual/core/src/internal/service-spec.ts index ee6d136bc..107fe3b5a 100644 --- a/packages/@eventual/core/src/internal/service-spec.ts +++ b/packages/@eventual/core/src/internal/service-spec.ts @@ -322,4 +322,5 @@ export interface QueueSpec { fifo: boolean; contentBasedDeduplication?: boolean; visibilityTimeout?: DurationSchedule; + delay?: DurationSchedule; } diff --git a/packages/@eventual/core/src/queue.ts b/packages/@eventual/core/src/queue.ts index 8a5aee13f..6b54eef65 100644 --- a/packages/@eventual/core/src/queue.ts +++ b/packages/@eventual/core/src/queue.ts @@ -5,9 +5,11 @@ import type { QueueSpec, SourceLocation, } from "./internal/service-spec.js"; -import type { DurationSchedule } from "./schedule.js"; +import { DurationSchedule, Schedule } from "./schedule.js"; import type { ServiceContext } from "./service.js"; +export const DEFAULT_QUEUE_VISIBILITY_TIMEOUT = Schedule.duration(30); + /** * Context passed to the handler. */ @@ -146,12 +148,18 @@ export type FifoQueueMessagePropertyReference = | ((message: Message) => string); interface QueueOptionsBase { - /** k + /** * The default visibility timeout for messages in the queue. * * @default Schedule.duration(30, "seconds") */ visibilityTimeout?: DurationSchedule; + /** + * Amount of time to delay the delivery of messages in the queue to consumers. + * + * @default 0 seconds + */ + delay?: DurationSchedule; fifo?: Fifo; handlerOptions?: QueueHandlerSpec; } @@ -209,13 +217,14 @@ export function queue< const [sourceLocation, name, options, handler] = _args.length === 4 ? _args : [undefined, ..._args]; - const { fifo, handlerOptions, visibilityTimeout } = options; + const { fifo, handlerOptions, visibilityTimeout, delay } = options; const queueBase = { kind: "Queue", name, fifo, visibilityTimeout, + delay, changeMessageVisibility(...args) { return getEventualHook().executeEventualCall( createCall(CallKind.QueueCall, { From 47d73dba0c89cfb6b38f1f103835de5b2399b621 Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Wed, 23 Aug 2023 09:07:37 -0500 Subject: [PATCH 15/20] suppor delay in local queue --- .../core-runtime/src/local/clients/queue-client.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/@eventual/core-runtime/src/local/clients/queue-client.ts b/packages/@eventual/core-runtime/src/local/clients/queue-client.ts index 963bb1b10..bfac978f6 100644 --- a/packages/@eventual/core-runtime/src/local/clients/queue-client.ts +++ b/packages/@eventual/core-runtime/src/local/clients/queue-client.ts @@ -170,11 +170,10 @@ export class LocalQueue { receiveCount: 0, }); - if (operation.delay) { - const visibilityTime = computeScheduleDate( - operation.delay, - operation.message - ); + const delay = operation.delay ?? this.queue.delay; + + if (delay) { + const visibilityTime = computeScheduleDate(delay, operation.message); this.messageVisibility[operation.message.id] = visibilityTime; // trigger an event to poll this queue for events on the queue when this message is planned to be visible. this.localConnector.scheduleEvent(visibilityTime, { From 9402ad3b621d04197bfefb97648789f4441ec262 Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Wed, 23 Aug 2023 09:28:22 -0500 Subject: [PATCH 16/20] support turning off SSE --- .../@eventual/aws-cdk/src/queue-service.ts | 4 ++++ .../@eventual/compiler/src/eventual-infer.ts | 1 + .../core/src/internal/service-spec.ts | 9 +++---- packages/@eventual/core/src/queue.ts | 24 ++++++++++++------- 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/packages/@eventual/aws-cdk/src/queue-service.ts b/packages/@eventual/aws-cdk/src/queue-service.ts index ff6e65d99..b6afcba25 100644 --- a/packages/@eventual/aws-cdk/src/queue-service.ts +++ b/packages/@eventual/aws-cdk/src/queue-service.ts @@ -148,6 +148,10 @@ class Queue extends Construct implements IQueue { computeDurationSeconds(props.queue.visibilityTimeout) ) : undefined, + // TODO: support customer managed key + encryption: props.queue.encryption + ? sqs.QueueEncryption.SQS_MANAGED + : sqs.QueueEncryption.UNENCRYPTED, ...overrides, }); diff --git a/packages/@eventual/compiler/src/eventual-infer.ts b/packages/@eventual/compiler/src/eventual-infer.ts index b64107c95..6372b195f 100644 --- a/packages/@eventual/compiler/src/eventual-infer.ts +++ b/packages/@eventual/compiler/src/eventual-infer.ts @@ -181,6 +181,7 @@ export function inferFromMemory(openApi: ServiceSpec["openApi"]): ServiceSpec { }, contentBasedDeduplication: q.contentBasedDeduplication, delay: q.delay, + encryption: q.encryption, visibilityTimeout: q.visibilityTimeout, } satisfies QueueSpec) ), diff --git a/packages/@eventual/core/src/internal/service-spec.ts b/packages/@eventual/core/src/internal/service-spec.ts index 107fe3b5a..d9ebf8ab8 100644 --- a/packages/@eventual/core/src/internal/service-spec.ts +++ b/packages/@eventual/core/src/internal/service-spec.ts @@ -317,10 +317,11 @@ export interface QueueHandlerSpec { } export interface QueueSpec { - name: Name; - handler: QueueHandlerSpec; - fifo: boolean; contentBasedDeduplication?: boolean; - visibilityTimeout?: DurationSchedule; delay?: DurationSchedule; + encryption: boolean; + fifo: boolean; + handler: QueueHandlerSpec; + name: Name; + visibilityTimeout?: DurationSchedule; } diff --git a/packages/@eventual/core/src/queue.ts b/packages/@eventual/core/src/queue.ts index 6b54eef65..5026e0864 100644 --- a/packages/@eventual/core/src/queue.ts +++ b/packages/@eventual/core/src/queue.ts @@ -148,20 +148,26 @@ export type FifoQueueMessagePropertyReference = | ((message: Message) => string); interface QueueOptionsBase { - /** - * The default visibility timeout for messages in the queue. - * - * @default Schedule.duration(30, "seconds") - */ - visibilityTimeout?: DurationSchedule; /** * Amount of time to delay the delivery of messages in the queue to consumers. * * @default 0 seconds */ delay?: DurationSchedule; + /** + * When true, the contents of the messages are encrypted server side with a managed key. + * + * @default true + */ + encryption?: boolean; fifo?: Fifo; handlerOptions?: QueueHandlerSpec; + /** + * The default visibility timeout for messages in the queue. + * + * @default Schedule.duration(30, "seconds") + */ + visibilityTimeout?: DurationSchedule; } export type QueueOptions = @@ -217,14 +223,16 @@ export function queue< const [sourceLocation, name, options, handler] = _args.length === 4 ? _args : [undefined, ..._args]; - const { fifo, handlerOptions, visibilityTimeout, delay } = options; + const { fifo, handlerOptions, visibilityTimeout, delay, encryption } = + options; const queueBase = { kind: "Queue", name, + delay, + encryption: encryption ?? true, fifo, visibilityTimeout, - delay, changeMessageVisibility(...args) { return getEventualHook().executeEventualCall( createCall(CallKind.QueueCall, { From 389952cf621ad19a13d842a4f376458e4e61bf4e Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Wed, 23 Aug 2023 12:54:16 -0500 Subject: [PATCH 17/20] send message batch and delete batch --- .../aws-runtime/src/clients/queue-client.ts | 66 ++++++- .../__snapshots__/infer-plugin.test.ts.snap | 41 +++- .../src/call-executors/queue-call-executor.ts | 6 +- .../core-runtime/src/clients/queue-client.ts | 29 ++- .../src/local/clients/queue-client.ts | 65 ++++++- packages/@eventual/core/src/internal/calls.ts | 38 +++- packages/@eventual/core/src/queue.ts | 178 +++++++++++++----- 7 files changed, 347 insertions(+), 76 deletions(-) diff --git a/packages/@eventual/aws-runtime/src/clients/queue-client.ts b/packages/@eventual/aws-runtime/src/clients/queue-client.ts index f7e0be159..f8d850323 100644 --- a/packages/@eventual/aws-runtime/src/clients/queue-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/queue-client.ts @@ -1,21 +1,28 @@ import { ChangeMessageVisibilityCommand, + DeleteMessageBatchCommand, DeleteMessageCommand, + SendMessageBatchCommand, SendMessageCommand, type SQSClient, } from "@aws-sdk/client-sqs"; import { isFifoContentBasedDeduplication, type DurationSchedule, + type QueueBatchResponse, + type QueueDeleteBatchEntry, } from "@eventual/core"; import { computeDurationSeconds, getLazy, - type LazyValue, QueueClient, + type LazyValue, type QueueProvider, } from "@eventual/core-runtime"; -import type { QueueSendMessageOperation } from "@eventual/core/internal"; +import type { + QueueSendMessageBatchOperation, + QueueSendMessageOperation, +} from "@eventual/core/internal"; import { queueServiceQueueName } from "../utils.js"; export interface QueueRuntimeOverrides { @@ -61,6 +68,42 @@ export class AWSQueueClient extends QueueClient { ); } + public override async sendMessageBatch( + operation: QueueSendMessageBatchOperation + ): Promise { + const result = await this.props.sqs.send( + new SendMessageBatchCommand({ + QueueUrl: this.physicalQueueUrl(operation.queueName), + Entries: operation.fifo + ? operation.entries.map((m) => ({ + Id: m.id, + MessageBody: JSON.stringify(m.message), + DelaySeconds: m.delay + ? computeDurationSeconds(m.delay) + : undefined, + MessageDeduplicationId: + // message deduplication is only supported for FIFO queues + // if fifo, deduplication id is required unless the definition asserts content based deduplication is on. + isFifoContentBasedDeduplication(m.messageDeduplicationId) + ? undefined + : m.messageDeduplicationId, + MessageGroupId: m.messageGroupId, + })) + : operation.entries.map((m) => ({ + Id: m.id, + MessageBody: JSON.stringify(m.message), + DelaySeconds: m.delay + ? computeDurationSeconds(m.delay) + : undefined, + })), + }) + ); + + return { + failed: result.Failed?.map((f) => ({ id: f.Id!, message: f.Message })), + }; + } + public override async changeMessageVisibility( queueName: string, receiptHandle: string, @@ -87,6 +130,25 @@ export class AWSQueueClient extends QueueClient { ); } + public override async deleteMessageBatch( + queueName: string, + entries: QueueDeleteBatchEntry[] + ): Promise { + const result = await this.props.sqs.send( + new DeleteMessageBatchCommand({ + QueueUrl: this.physicalQueueUrl(queueName), + Entries: entries.map((e) => ({ + Id: e.id, + ReceiptHandle: e.receiptHandle, + })), + }) + ); + + return { + failed: result.Failed?.map((f) => ({ id: f.Id!, message: f.Message })), + }; + } + public physicalQueueUrl(queueName: string) { return `https://sqs.${getLazy( this.props.awsRegion diff --git a/packages/@eventual/compiler/test/__snapshots__/infer-plugin.test.ts.snap b/packages/@eventual/compiler/test/__snapshots__/infer-plugin.test.ts.snap index aa237a11c..613e18d58 100644 --- a/packages/@eventual/compiler/test/__snapshots__/infer-plugin.test.ts.snap +++ b/packages/@eventual/compiler/test/__snapshots__/infer-plugin.test.ts.snap @@ -44,18 +44,36 @@ var CallKind; CallKind2[CallKind2["TaskRequestCall"] = 12] = "TaskRequestCall"; CallKind2[CallKind2["SearchCall"] = 11] = "SearchCall"; CallKind2[CallKind2["StartWorkflowCall"] = 13] = "StartWorkflowCall"; + CallKind2[CallKind2["QueueCall"] = 15] = "QueueCall"; })(CallKind || (CallKind = {})); +var Schedule = { + duration(dur, unit = "seconds") { + return { + type: "Duration", + dur, + unit + }; + }, + time(isoDate) { + return { + type: "Time", + isoDate: typeof isoDate === "string" ? isoDate : isoDate.toISOString() + }; + } +}; + var PropertyKind; (function(PropertyKind2) { PropertyKind2[PropertyKind2["BucketPhysicalName"] = 0] = "BucketPhysicalName"; - PropertyKind2[PropertyKind2["OpenSearchClient"] = 1] = "OpenSearchClient"; - PropertyKind2[PropertyKind2["ServiceClient"] = 2] = "ServiceClient"; - PropertyKind2[PropertyKind2["ServiceName"] = 3] = "ServiceName"; - PropertyKind2[PropertyKind2["ServiceSpec"] = 4] = "ServiceSpec"; - PropertyKind2[PropertyKind2["ServiceType"] = 5] = "ServiceType"; - PropertyKind2[PropertyKind2["ServiceUrl"] = 6] = "ServiceUrl"; - PropertyKind2[PropertyKind2["TaskToken"] = 7] = "TaskToken"; + PropertyKind2[PropertyKind2["QueuePhysicalName"] = 1] = "QueuePhysicalName"; + PropertyKind2[PropertyKind2["OpenSearchClient"] = 2] = "OpenSearchClient"; + PropertyKind2[PropertyKind2["ServiceClient"] = 3] = "ServiceClient"; + PropertyKind2[PropertyKind2["ServiceName"] = 4] = "ServiceName"; + PropertyKind2[PropertyKind2["ServiceSpec"] = 5] = "ServiceSpec"; + PropertyKind2[PropertyKind2["ServiceType"] = 6] = "ServiceType"; + PropertyKind2[PropertyKind2["ServiceUrl"] = 7] = "ServiceUrl"; + PropertyKind2[PropertyKind2["TaskToken"] = 8] = "TaskToken"; })(PropertyKind || (PropertyKind = {})); globalThis._eventual ??= { resources: {} }; @@ -73,11 +91,12 @@ function isSourceLocation(a) { var ServiceType; (function(ServiceType2) { + ServiceType2["BucketNotificationHandlerWorker"] = "BucketNotificationHandlerWorker"; ServiceType2["CommandWorker"] = "CommandWorker"; - ServiceType2["Subscription"] = "Subscription"; - ServiceType2["OrchestratorWorker"] = "OrchestratorWorker"; ServiceType2["EntityStreamWorker"] = "EntityStreamWorker"; - ServiceType2["BucketNotificationHandlerWorker"] = "BucketNotificationHandlerWorker"; + ServiceType2["OrchestratorWorker"] = "OrchestratorWorker"; + ServiceType2["QueueHandlerWorker"] = "QueueHandlerWorker"; + ServiceType2["Subscription"] = "Subscription"; ServiceType2["TaskWorker"] = "TaskWorker"; ServiceType2["TransactionWorker"] = "TransactionWorker"; })(ServiceType || (ServiceType = {})); @@ -264,6 +283,8 @@ var LogLevel; })(LogLevel || (LogLevel = {})); var LOG_LEVELS = Object.values(LogLevel); +var DEFAULT_QUEUE_VISIBILITY_TIMEOUT = Schedule.duration(30); + var _BaseCachingSecret_value; _BaseCachingSecret_value = /* @__PURE__ */ new WeakMap(); diff --git a/packages/@eventual/core-runtime/src/call-executors/queue-call-executor.ts b/packages/@eventual/core-runtime/src/call-executors/queue-call-executor.ts index 8c267dfb3..395cf3b52 100644 --- a/packages/@eventual/core-runtime/src/call-executors/queue-call-executor.ts +++ b/packages/@eventual/core-runtime/src/call-executors/queue-call-executor.ts @@ -1,13 +1,15 @@ import { QueueCall, isQueueOperationOfType } from "@eventual/core/internal"; import type { CallExecutor } from "../call-executor.js"; -import { QueueClient } from "../clients/queue-client.js"; +import type { QueueClient } from "../clients/queue-client.js"; export class QueueCallExecutor implements CallExecutor { constructor(private queueClient: QueueClient) {} - public execute(call: QueueCall): Promise { + public execute(call: QueueCall) { const operation = call.operation; if (isQueueOperationOfType("sendMessage", operation)) { return this.queueClient.sendMessage(operation); + } else if (isQueueOperationOfType("sendMessageBatch", operation)) { + return this.queueClient.sendMessageBatch(operation); } return this.queueClient[operation.operation]( call.operation.queueName, diff --git a/packages/@eventual/core-runtime/src/clients/queue-client.ts b/packages/@eventual/core-runtime/src/clients/queue-client.ts index 79f9e6dab..ecb7df70b 100644 --- a/packages/@eventual/core-runtime/src/clients/queue-client.ts +++ b/packages/@eventual/core-runtime/src/clients/queue-client.ts @@ -1,13 +1,22 @@ -import type { DurationSchedule, FifoQueue } from "@eventual/core"; -import { Queue } from "@eventual/core"; -import { +import type { + DurationSchedule, + FifoQueue, + Queue, + QueueBatchResponse, + QueueDeleteBatchEntry, +} from "@eventual/core"; +import type { QueueMethod, + QueueSendMessageBatchOperation, QueueSendMessageOperation, } from "@eventual/core/internal"; import type { QueueProvider } from "../providers/queue-provider.js"; type QueueClientBase = { - [K in keyof Pick>]: ( + [K in keyof Pick< + Queue, + Exclude + >]: ( queueName: string, ...args: Parameters ) => ReturnType; @@ -15,6 +24,9 @@ type QueueClientBase = { sendMessage: ( operation: QueueSendMessageOperation ) => ReturnType; + sendMessageBatch: ( + operation: QueueSendMessageBatchOperation + ) => ReturnType; }; export abstract class QueueClient implements QueueClientBase { @@ -24,6 +36,10 @@ export abstract class QueueClient implements QueueClientBase { operation: QueueSendMessageOperation ): Promise; + public abstract sendMessageBatch( + operation: QueueSendMessageBatchOperation + ): Promise; + public abstract changeMessageVisibility( queueName: string, receiptHandle: string, @@ -35,6 +51,11 @@ export abstract class QueueClient implements QueueClientBase { receiptHandle: string ): Promise; + public abstract deleteMessageBatch( + queueName: string, + entries: QueueDeleteBatchEntry[] + ): Promise; + public abstract physicalName(queueName: string): string; protected getQueue(queueName: string) { diff --git a/packages/@eventual/core-runtime/src/local/clients/queue-client.ts b/packages/@eventual/core-runtime/src/local/clients/queue-client.ts index bfac978f6..ced9a029e 100644 --- a/packages/@eventual/core-runtime/src/local/clients/queue-client.ts +++ b/packages/@eventual/core-runtime/src/local/clients/queue-client.ts @@ -1,19 +1,22 @@ import { DEFAULT_QUEUE_VISIBILITY_TIMEOUT, + QueueBatchResponse, type DurationSchedule, type FifoQueue, type FifoQueueHandlerMessageItem, type Queue, type StandardQueueHandlerMessageItem, + QueueDeleteBatchEntry, } from "@eventual/core"; -import type { QueueSendMessageOperation } from "@eventual/core/internal"; +import type { + QueueSendMessageBatchOperation, + QueueSendMessageOperation, +} from "@eventual/core/internal"; import { ulid } from "ulidx"; import { QueueClient } from "../../clients/queue-client.js"; -import { - computeScheduleDate, - type LocalEnvConnector, - type QueueProvider, -} from "../../index.js"; +import type { QueueProvider } from "../../providers/queue-provider.js"; +import { computeScheduleDate } from "../../schedule.js"; +import type { LocalEnvConnector } from "../local-container.js"; export interface QueueRetrieveMessagesRequest { maxMessages?: number; @@ -53,6 +56,44 @@ export class LocalQueueClient extends QueueClient { return queue.sendMessage(operation); } + public async sendMessageBatch( + operation: QueueSendMessageBatchOperation + ): Promise { + let queue = this.queues.get(operation.queueName); + if (!queue) { + queue = new LocalQueue( + this.queueProvider.getQueue(operation.queueName)!, + this.localConnector + ); + this.queues.set(operation.queueName, queue); + } + await Promise.all( + operation.fifo + ? operation.entries.map((m) => + queue!.sendMessage({ + fifo: operation.fifo, + delay: m.delay, + message: m.message, + messageDeduplicationId: m.messageDeduplicationId, + messageGroupId: m.messageGroupId, + queueName: operation.queueName, + operation: "sendMessage", + }) + ) + : operation.entries.map((m) => + queue!.sendMessage({ + fifo: false, + message: m.message, + queueName: operation.queueName, + delay: m.delay, + operation: "sendMessage", + }) + ) + ); + // messages cannot failed to be sent? + return {}; + } + public async changeMessageVisibility( queueName: string, receiptHandle: string, @@ -70,6 +111,18 @@ export class LocalQueueClient extends QueueClient { await queue?.deleteMessage(receiptHandle); } + public async deleteMessageBatch( + queueName: string, + entries: QueueDeleteBatchEntry[] + ): Promise { + const queue = this.queues.get(queueName); + await Promise.all( + entries.map((e) => queue?.deleteMessage(e.receiptHandle)) + ); + // messages cannot failed to be deleted? + return {}; + } + public physicalName(queueName: string): string { return queueName; } diff --git a/packages/@eventual/core/src/internal/calls.ts b/packages/@eventual/core/src/internal/calls.ts index 2b12c285c..43d5697b3 100644 --- a/packages/@eventual/core/src/internal/calls.ts +++ b/packages/@eventual/core/src/internal/calls.ts @@ -245,23 +245,41 @@ interface QueueOperationBase { operation: Op; } -export type QueueSendMessageOperation = QueueOperationBase<"sendMessage"> & { +export type StandardQueueSendMessagePayload = { delay?: DurationSchedule; message: any; -} & ( - | { - fifo: true; - messageGroupId: string; - messageDeduplicationId: string | FifoContentBasedDeduplication; - } - | { - fifo: false; - } +}; + +export interface FifoQueueSendMessagePayload + extends StandardQueueSendMessagePayload { + messageGroupId: string; + messageDeduplicationId: string | FifoContentBasedDeduplication; +} + +export type QueueSendMessageOperation = QueueOperationBase<"sendMessage"> & + ( + | ({ fifo: true } & FifoQueueSendMessagePayload) + | ({ fifo: false } & StandardQueueSendMessagePayload) ); +export type QueueSendMessageBatchOperation = + QueueOperationBase<"sendMessageBatch"> & + ( + | { + fifo: true; + entries: ({ id: string } & FifoQueueSendMessagePayload)[]; + } + | { + fifo: false; + entries: ({ id: string } & StandardQueueSendMessagePayload)[]; + } + ); + export type QueueOperation = Op extends "sendMessage" ? QueueSendMessageOperation + : Op extends "sendMessageBatch" + ? QueueSendMessageBatchOperation : QueueOperationBase & { params: Parameters; }; diff --git a/packages/@eventual/core/src/queue.ts b/packages/@eventual/core/src/queue.ts index 5026e0864..1228474a1 100644 --- a/packages/@eventual/core/src/queue.ts +++ b/packages/@eventual/core/src/queue.ts @@ -1,4 +1,9 @@ -import { CallKind, createCall, type QueueCall } from "./internal/calls.js"; +import { + CallKind, + createCall, + FifoQueueSendMessagePayload, + type QueueCall, +} from "./internal/calls.js"; import { registerEventualResource } from "./internal/resources.js"; import type { QueueHandlerSpec, @@ -91,6 +96,27 @@ export interface FifoQueueSendMessageOptions dedupeId?: string; } +export interface QueueSendMessageBatchEntry< + Message, + Options extends StandardQueueSendMessageOptions = StandardQueueSendMessageOptions +> { + /** + * ID of the message. Will be returned if the message fails to send. + */ + id: string; + message: Message; + options?: Options; +} + +export interface QueueBatchResponse { + failed?: { id: string; message?: string }[]; +} + +export interface QueueDeleteBatchEntry { + id: string; + receiptHandle: string; +} + interface QueueBase extends Omit, "handler" | "message"> { kind: "Queue"; @@ -100,6 +126,9 @@ interface QueueBase timeout: DurationSchedule ): Promise; deleteMessage(receiptHandle: string): Promise; + deleteMessageBatch( + entries: QueueDeleteBatchEntry[] + ): Promise; } export interface StandardQueue @@ -110,6 +139,9 @@ export interface StandardQueue message: Message, options?: StandardQueueSendMessageOptions ): Promise; + sendMessageBatch( + entries: QueueSendMessageBatchEntry[] + ): Promise; } export interface FifoQueue @@ -124,6 +156,9 @@ export interface FifoQueue message: Message, options?: FifoQueueSendMessageOptions ): Promise; + sendMessageBatch( + entries: QueueSendMessageBatchEntry[] + ): Promise; } export function isFifoQueue(queue: Queue): queue is FifoQueue { @@ -235,7 +270,7 @@ export function queue< visibilityTimeout, changeMessageVisibility(...args) { return getEventualHook().executeEventualCall( - createCall(CallKind.QueueCall, { + createCall>(CallKind.QueueCall, { operation: { queueName: name, operation: "changeMessageVisibility", @@ -246,7 +281,7 @@ export function queue< }, deleteMessage(...args) { return getEventualHook().executeEventualCall( - createCall(CallKind.QueueCall, { + createCall>(CallKind.QueueCall, { operation: { queueName: name, operation: "deleteMessage", @@ -255,6 +290,17 @@ export function queue< }) ); }, + deleteMessageBatch(...args) { + return getEventualHook().executeEventualCall( + createCall>(CallKind.QueueCall, { + operation: { + queueName: name, + operation: "deleteMessageBatch", + params: args, + }, + }) + ); + }, } satisfies Partial>; const queue: Queue = fifo @@ -275,51 +321,35 @@ export function queue< message: Message, sendOptions?: FifoQueueSendMessageOptions ) { - const messageGroupIdReference = options?.groupBy; - const messageDedupeIdReference = options?.dedupe; - - const messageGroupId = - sendOptions?.group ?? messageGroupIdReference - ? typeof messageGroupIdReference === "string" - ? message[messageGroupIdReference] - : messageGroupIdReference?.(message) - : undefined; - if (!messageGroupId || typeof messageGroupId !== "string") { - throw new Error( - "Message Group Id must be provided and must be a non-empty string" - ); - } - - const messageDeduplicationId = - sendOptions?.dedupeId ?? messageDedupeIdReference - ? typeof messageDedupeIdReference === "string" - ? message[messageDedupeIdReference] - : typeof messageDedupeIdReference === "function" - ? messageDedupeIdReference?.(message) - : messageDedupeIdReference - : undefined; - if ( - !messageDeduplicationId || - !( - typeof messageDeduplicationId === "string" || - isFifoContentBasedDeduplication(messageDeduplicationId) - ) - ) { - throw new Error( - "Message Deduplication Id must be provided and a non-empty string or set to { contentBasedDeduplication: true }" - ); - } - return getEventualHook().executeEventualCall( createCall(CallKind.QueueCall, { operation: { queueName: name, operation: "sendMessage", fifo, - messageGroupId, - messageDeduplicationId, - message, - delay: sendOptions?.delay, + ...processFifoSendMessageMessage(message, options, sendOptions), + }, + }) + ); + }, + sendMessageBatch( + entries: QueueSendMessageBatchEntry< + Message, + FifoQueueSendMessageOptions + >[] + ) { + const processMessages = entries.map((m) => ({ + id: m.id, + ...processFifoSendMessageMessage(m.message, options, m.options), + })); + + return getEventualHook().executeEventualCall( + createCall>(CallKind.QueueCall, { + operation: { + queueName: name, + operation: "sendMessageBatch", + fifo, + entries: processMessages, }, }) ); @@ -349,6 +379,22 @@ export function queue< }) ); }, + sendMessageBatch(messages) { + return getEventualHook().executeEventualCall( + createCall>(CallKind.QueueCall, { + operation: { + queueName: name, + operation: "sendMessageBatch", + fifo: false, + entries: messages.map((m) => ({ + id: m.id, + message: m.message, + delay: m.options?.delay, + })), + }, + }) + ); + }, }; return registerEventualResource("Queue", queue as Queue); @@ -368,3 +414,51 @@ export function isFifoContentBasedDeduplication( ): value is FifoContentBasedDeduplication { return value && typeof value === "object" && value.contentBasedDeduplication; } + +function processFifoSendMessageMessage( + message: any, + options: FifoQueueOptions, + sendOptions?: FifoQueueSendMessageOptions +): FifoQueueSendMessagePayload { + const messageGroupIdReference = options?.groupBy; + const messageDedupeIdReference = options?.dedupe; + + const messageGroupId = + sendOptions?.group ?? messageGroupIdReference + ? typeof messageGroupIdReference === "string" + ? message[messageGroupIdReference] + : messageGroupIdReference?.(message) + : undefined; + if (!messageGroupId || typeof messageGroupId !== "string") { + throw new Error( + "Message Group Id must be provided and must be a non-empty string" + ); + } + + const messageDeduplicationId = + sendOptions?.dedupeId ?? messageDedupeIdReference + ? typeof messageDedupeIdReference === "string" + ? message[messageDedupeIdReference] + : typeof messageDedupeIdReference === "function" + ? messageDedupeIdReference?.(message) + : messageDedupeIdReference + : undefined; + if ( + !messageDeduplicationId || + !( + typeof messageDeduplicationId === "string" || + isFifoContentBasedDeduplication(messageDeduplicationId) + ) + ) { + throw new Error( + "Message Deduplication Id must be provided and a non-empty string or set to { contentBasedDeduplication: true }" + ); + } + + return { + messageGroupId, + messageDeduplicationId, + message, + delay: sendOptions?.delay, + }; +} From a95bab0cca68f326c8c5360156ef1163185e03b2 Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Wed, 23 Aug 2023 16:43:24 -0500 Subject: [PATCH 18/20] woops --- .../@eventual/core-runtime/src/handlers/entity-stream-worker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@eventual/core-runtime/src/handlers/entity-stream-worker.ts b/packages/@eventual/core-runtime/src/handlers/entity-stream-worker.ts index 11a0acb33..1e0f5318c 100644 --- a/packages/@eventual/core-runtime/src/handlers/entity-stream-worker.ts +++ b/packages/@eventual/core-runtime/src/handlers/entity-stream-worker.ts @@ -49,7 +49,7 @@ export function createEntityStreamWorker( return JSON.stringify(normalizedKey); }, async (item) => { - const result = streamHandler.handler(item, context); + const result = await streamHandler.handler(item, context); if (result === false) { throw new Error("Handler failed"); } From 61d1cfefb41882a595a65ee1f59ab6692f2040a0 Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Mon, 28 Aug 2023 10:44:31 -0500 Subject: [PATCH 19/20] re-add comma --- packages/@eventual/core-runtime/src/stores/entity-store.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@eventual/core-runtime/src/stores/entity-store.ts b/packages/@eventual/core-runtime/src/stores/entity-store.ts index e5e507d6d..785dde676 100644 --- a/packages/@eventual/core-runtime/src/stores/entity-store.ts +++ b/packages/@eventual/core-runtime/src/stores/entity-store.ts @@ -30,7 +30,7 @@ import { keyHasInlineBetween, type EntityMethod, type KeyDefinition, - type KeyDefinitionPart + type KeyDefinitionPart, } from "@eventual/core/internal"; import type { EntityProvider } from "../providers/entity-provider.js"; From cc24252a6cd566db572f50167d047bf2f3c157cd Mon Sep 17 00:00:00 2001 From: Sam Sussman Date: Mon, 28 Aug 2023 12:11:45 -0500 Subject: [PATCH 20/20] fix(local): remove local ambigous event order issue --- .../core-runtime/src/local/time-controller.ts | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/packages/@eventual/core-runtime/src/local/time-controller.ts b/packages/@eventual/core-runtime/src/local/time-controller.ts index 3895a5eba..a0799f4ab 100644 --- a/packages/@eventual/core-runtime/src/local/time-controller.ts +++ b/packages/@eventual/core-runtime/src/local/time-controller.ts @@ -21,8 +21,16 @@ export interface TimeEvent { export class TimeController { // current millisecond time private current = 0; + private ordCounter = 0; private increment: number; - private timeHeap: Heap>; + private timeHeap: Heap< + TimeEvent & { + /** + * Order the event was added to the heap. Acts as a tie break when events have the same time. + */ + ord: number; + } + >; constructor( initialEvents: TimeEvent[], @@ -30,8 +38,16 @@ export class TimeController { ) { this.current = props?.start ?? 0; this.increment = props?.increment ?? 1; - this.timeHeap = new Heap>((a, b) => a.timestamp - b.timestamp); - this.timeHeap.init(initialEvents); + this.timeHeap = new Heap & { ord: number }>((a, b) => { + const diff = a.timestamp - b.timestamp; + if (diff === 0) { + return a.ord - b.ord; + } + return diff; + }); + this.timeHeap.init( + initialEvents.map((e) => ({ ...e, ord: this.ordCounter++ })) + ); } /** @@ -121,21 +137,23 @@ export class TimeController { * Add an event to the {@link TimeController}. */ public addEvent(timestamp: number, event: E): void { - this.timeHeap.add({ timestamp, event }); + this.timeHeap.add({ timestamp, event, ord: this.ordCounter++ }); } /** * Add an event to the {@link TimeController}. */ public addEventAtNextTick(event: E): void { - this.timeHeap.add({ timestamp: this.nextTick, event }); + this.addEvent(this.nextTick, event); } /** * Add events to the {@link TimeController}. */ public addEvents(timeEvents: TimeEvent[]): void { - this.timeHeap.addAll(timeEvents); + this.timeHeap.addAll( + timeEvents.map((e) => ({ ...e, ord: this.ordCounter++ })) + ); } /**