From b853d98f7f18cc0035c294ad267917b800b98cdb Mon Sep 17 00:00:00 2001 From: Felipe Forbeck Date: Fri, 11 Oct 2024 14:01:19 -0300 Subject: [PATCH 1/3] feat: usage/record capability definition --- packages/capabilities/src/types.ts | 15 +++++ packages/capabilities/src/usage.js | 21 ++++++- packages/upload-api/src/types/usage.ts | 8 +++ packages/upload-api/src/usage.js | 8 ++- packages/upload-api/src/usage/record.js | 24 ++++++++ .../upload-api/test/storage/usage-storage.js | 29 ++++++++++ packages/upload-client/src/types.ts | 8 +++ packages/w3up-client/src/capability/usage.js | 55 +++++++++++++++++++ packages/w3up-client/src/types.ts | 4 ++ .../w3up-client/test/capability/usage.test.js | 42 ++++++++++++++ 10 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 packages/upload-api/src/usage/record.js diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index ba28eeb05..ce32714ed 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -131,6 +131,10 @@ export type UsageReport = InferInvokedCapability export type UsageReportSuccess = Record export type UsageReportFailure = Ucanto.Failure +export type EgressRecord = InferInvokedCapability +export type EgressRecordSuccess = Unit +export type EgressRecordFailure = Ucanto.Failure + export interface UsageData { /** Provider the report concerns, e.g. `did:web:web3.storage` */ provider: ProviderDID @@ -161,6 +165,17 @@ export interface UsageData { }> } +export interface EgressData { + /** Id of the customer that is being billed. */ + customer: AccountDID + /** CID of the resource that was served. */ + resourceCID: string + /** Amount of bytes served. */ + bytes: number + /** ISO datetime that the bytes were served at. */ + servedAt: ISO8601Date +} + // Provider export type ProviderAdd = InferInvokedCapability // eslint-disable-next-line @typescript-eslint/no-empty-interface diff --git a/packages/capabilities/src/usage.js b/packages/capabilities/src/usage.js index d80fb212d..b37c7ffc0 100644 --- a/packages/capabilities/src/usage.js +++ b/packages/capabilities/src/usage.js @@ -1,5 +1,5 @@ import { capability, ok, Schema } from '@ucanto/validator' -import { and, equal, equalWith, SpaceDID } from './utils.js' +import { AccountDID, and, equal, equalWith, SpaceDID } from './utils.js' /** * Capability can only be delegated (but not invoked) allowing audience to @@ -40,3 +40,22 @@ export const report = capability({ ) }, }) + +/** + * Capability can be invoked by an agent to record usage data for a given resource. + */ +export const record = capability({ + can: 'usage/record', + with: SpaceDID, + nb: Schema.struct({ + /** MailTo DID of the customer that is being billed. */ + customer: AccountDID, + /** CID of the resource that was served. */ + resourceCID: Schema.string(), + /** Amount of bytes served. */ + bytes: Schema.integer().greaterThan(0), + /** Timestamp of the event in seconds after Unix epoch. */ + servedAt: Schema.integer().greaterThan(-1), + }), + derives: equalWith, +}) diff --git a/packages/upload-api/src/types/usage.ts b/packages/upload-api/src/types/usage.ts index bed076201..b1aa7b5d5 100644 --- a/packages/upload-api/src/types/usage.ts +++ b/packages/upload-api/src/types/usage.ts @@ -3,6 +3,8 @@ import { ProviderDID, SpaceDID, UsageData, + EgressData, + AccountDID, } from '@web3-storage/capabilities/types' export type { UsageData } @@ -13,4 +15,10 @@ export interface UsageStorage { space: SpaceDID, period: { from: Date; to: Date } ) => Promise> + record: ( + customer: AccountDID, + resourceCID: string, + bytes: number, + servedAt: Date + ) => Promise> } diff --git a/packages/upload-api/src/usage.js b/packages/upload-api/src/usage.js index 04076baca..8cea68010 100644 --- a/packages/upload-api/src/usage.js +++ b/packages/upload-api/src/usage.js @@ -1,4 +1,8 @@ -import { provide } from './usage/report.js' +import { provide as provideReport } from './usage/report.js' +import { provide as provideRecord } from './usage/record.js' /** @param {import('./types.js').UsageServiceContext} context */ -export const createService = (context) => ({ report: provide(context) }) +export const createService = (context) => ({ + report: provideReport(context), + record: provideRecord(context), +}) diff --git a/packages/upload-api/src/usage/record.js b/packages/upload-api/src/usage/record.js new file mode 100644 index 000000000..2bb8026e1 --- /dev/null +++ b/packages/upload-api/src/usage/record.js @@ -0,0 +1,24 @@ +import * as API from '../types.js' +import * as Provider from '@ucanto/server' +import { Usage } from '@web3-storage/capabilities' + +/** @param {API.UsageServiceContext} context */ +export const provide = (context) => + Provider.provide(Usage.record, (input) => record(input, context)) + +/** + * @param {API.Input} input + * @param {API.UsageServiceContext} context + * @returns {Promise>} + */ +const record = async ({ capability }, context) => { + const res = await context.usageStorage.record( + capability.nb.customer, + capability.nb.resourceCID, + capability.nb.bytes, + new Date(capability.nb.servedAt * 1000) + ) + if (res.error) return res + + return res +} diff --git a/packages/upload-api/test/storage/usage-storage.js b/packages/upload-api/test/storage/usage-storage.js index 706bc3d5b..d15871ff0 100644 --- a/packages/upload-api/test/storage/usage-storage.js +++ b/packages/upload-api/test/storage/usage-storage.js @@ -9,6 +9,10 @@ export class UsageStorage { constructor(storeTable, allocationsStorage) { this.storeTable = storeTable this.allocationsStorage = allocationsStorage + /** + * @type {Record} + */ + this._egressRecords = {} } get items() { @@ -64,4 +68,29 @@ export class UsageStorage { }, } } + + /** + * Simulate a record of egress data for a customer. + * + * @param {import('../types.js').AccountDID} customer + * @param {string} resourceCID + * @param {number} bytes + * @param {Date} servedAt + */ + async record(customer, resourceCID, bytes, servedAt) { + const egressData = { + customer, + resourceCID, + bytes, + servedAt: servedAt.toISOString(), + } + this._egressRecords[customer] = egressData + return Promise.resolve({ + ok: egressData, + }) + } + + get egressRecords() { + return this._egressRecords + } } diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts index 38a41f2f0..8d95612d6 100644 --- a/packages/upload-client/src/types.ts +++ b/packages/upload-client/src/types.ts @@ -69,6 +69,10 @@ import { UsageReport, UsageReportSuccess, UsageReportFailure, + EgressData, + EgressRecord, + EgressRecordSuccess, + EgressRecordFailure, ServiceAbility, } from '@web3-storage/capabilities/types' import { StorefrontService } from '@web3-storage/filecoin-client/storefront' @@ -135,6 +139,10 @@ export type { UsageReport, UsageReportSuccess, UsageReportFailure, + EgressData, + EgressRecord, + EgressRecordSuccess, + EgressRecordFailure, ListResponse, CARLink, PieceLink, diff --git a/packages/w3up-client/src/capability/usage.js b/packages/w3up-client/src/capability/usage.js index 19b53ba7c..fb31ab86d 100644 --- a/packages/w3up-client/src/capability/usage.js +++ b/packages/w3up-client/src/capability/usage.js @@ -31,6 +31,32 @@ export class UsageClient extends Base { return out.ok } + + /** + * Record egress data for the customer and served resource. + * + * Required delegated capabilities: + * - `usage/record` + * + * @param {import('../types.js').SpaceDID} space + * @param {API.EgressData} egressData + * @param {object} [options] + * @param {string} [options.nonce] + */ + async record(space, egressData, options) { + const out = await record( + { agent: this.agent }, + { ...options, space, egressData } + ) + /* c8 ignore next 7 */ + if (!out.ok) { + throw new Error(`failed ${UsageCapabilities.record.can} invocation`, { + cause: out.error, + }) + } + + return out.ok + } } /** @@ -61,3 +87,32 @@ export const report = async ( }) return receipt.out } + +/** + * Record egress data for the customer and served resource. + * + * @param {{agent: API.Agent}} client + * @param {object} options + * @param {API.SpaceDID} options.space + * @param {API.EgressData} options.egressData - + * @param {string} [options.nonce] + * @param {API.Delegation[]} [options.proofs] + * @returns {Promise>} + */ +export const record = async ( + { agent }, + { space, egressData, nonce, proofs = [] } +) => { + const receipt = await agent.invokeAndExecute(UsageCapabilities.record, { + with: space, + proofs, + nonce, + nb: { + customer: egressData.customer, + resourceCID: egressData.resourceCID, + bytes: egressData.bytes, + servedAt: Math.floor(new Date(egressData.servedAt).getTime() / 1000), + }, + }) + return receipt.out +} diff --git a/packages/w3up-client/src/types.ts b/packages/w3up-client/src/types.ts index 1480715fa..553c55e93 100644 --- a/packages/w3up-client/src/types.ts +++ b/packages/w3up-client/src/types.ts @@ -141,6 +141,10 @@ export type { UploadListItem, UsageReportSuccess, UsageReportFailure, + EgressData, + EgressRecord, + EgressRecordSuccess, + EgressRecordFailure, ListResponse, AnyLink, CARLink, diff --git a/packages/w3up-client/test/capability/usage.test.js b/packages/w3up-client/test/capability/usage.test.js index 13d598dd3..5a998dec5 100644 --- a/packages/w3up-client/test/capability/usage.test.js +++ b/packages/w3up-client/test/capability/usage.test.js @@ -2,6 +2,7 @@ import { AgentData } from '@web3-storage/access/agent' import { Client } from '../../src/client.js' import * as Test from '../test.js' import { receiptsEndpoint } from '../helpers/utils.js' +import { randomCAR } from '../helpers/random.js' export const UsageClient = Test.withContext({ report: { @@ -68,6 +69,47 @@ export const UsageClient = Test.withContext({ assert.deepEqual(report, {}) }, }, + record: { + 'should record egress': async ( + assert, + { connection, provisionsStorage } + ) => { + const alice = new Client(await AgentData.create(), { + // @ts-ignore + serviceConf: { + access: connection, + upload: connection, + }, + }) + + const space = await alice.createSpace('test') + const auth = await space.createAuthorization(alice) + await alice.addSpace(auth) + + // Then we setup a billing for this account + await provisionsStorage.put({ + // @ts-expect-error + provider: connection.id.did(), + account: alice.agent.did(), + consumer: space.did(), + }) + + const car = await randomCAR(128) + const resourceCID = car.cid + await alice.capability.upload.add(car.roots[0], [resourceCID]) + + const result = await alice.capability.upload.get(car.roots[0]) + assert.ok(result) + + const record = await alice.capability.usage.record(space.did(), { + customer: 'did:mailto:alice@web.mail', + resourceCID: resourceCID.link().toString(), + bytes: car.size, + servedAt: new Date().toISOString(), + }) + assert.ok(record) + }, + }, }) Test.test({ UsageClient }) From c91992e5c91b2d3c674dab53d11489ad953e1ba9 Mon Sep 17 00:00:00 2001 From: Felipe Forbeck Date: Mon, 14 Oct 2024 12:47:34 -0300 Subject: [PATCH 2/3] address review suggestions --- packages/capabilities/src/types.ts | 2 +- packages/capabilities/src/usage.js | 2 +- packages/upload-api/src/types/usage.ts | 4 ++-- packages/upload-api/src/usage/record.js | 2 +- packages/upload-api/test/storage/usage-storage.js | 6 +++--- packages/w3up-client/src/capability/usage.js | 4 ++-- packages/w3up-client/test/capability/usage.test.js | 6 +++--- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index ce32714ed..66c5794d4 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -169,7 +169,7 @@ export interface EgressData { /** Id of the customer that is being billed. */ customer: AccountDID /** CID of the resource that was served. */ - resourceCID: string + resource: UnknownLink /** Amount of bytes served. */ bytes: number /** ISO datetime that the bytes were served at. */ diff --git a/packages/capabilities/src/usage.js b/packages/capabilities/src/usage.js index b37c7ffc0..675b3dd64 100644 --- a/packages/capabilities/src/usage.js +++ b/packages/capabilities/src/usage.js @@ -51,7 +51,7 @@ export const record = capability({ /** MailTo DID of the customer that is being billed. */ customer: AccountDID, /** CID of the resource that was served. */ - resourceCID: Schema.string(), + resource: Schema.link(), /** Amount of bytes served. */ bytes: Schema.integer().greaterThan(0), /** Timestamp of the event in seconds after Unix epoch. */ diff --git a/packages/upload-api/src/types/usage.ts b/packages/upload-api/src/types/usage.ts index b1aa7b5d5..cfab49407 100644 --- a/packages/upload-api/src/types/usage.ts +++ b/packages/upload-api/src/types/usage.ts @@ -1,4 +1,4 @@ -import { Failure, Result } from '@ucanto/interface' +import { Failure, Result, UnknownLink } from '@ucanto/interface' import { ProviderDID, SpaceDID, @@ -17,7 +17,7 @@ export interface UsageStorage { ) => Promise> record: ( customer: AccountDID, - resourceCID: string, + resource: UnknownLink, bytes: number, servedAt: Date ) => Promise> diff --git a/packages/upload-api/src/usage/record.js b/packages/upload-api/src/usage/record.js index 2bb8026e1..b99955f07 100644 --- a/packages/upload-api/src/usage/record.js +++ b/packages/upload-api/src/usage/record.js @@ -14,7 +14,7 @@ export const provide = (context) => const record = async ({ capability }, context) => { const res = await context.usageStorage.record( capability.nb.customer, - capability.nb.resourceCID, + capability.nb.resource, capability.nb.bytes, new Date(capability.nb.servedAt * 1000) ) diff --git a/packages/upload-api/test/storage/usage-storage.js b/packages/upload-api/test/storage/usage-storage.js index d15871ff0..9415d8ba6 100644 --- a/packages/upload-api/test/storage/usage-storage.js +++ b/packages/upload-api/test/storage/usage-storage.js @@ -73,14 +73,14 @@ export class UsageStorage { * Simulate a record of egress data for a customer. * * @param {import('../types.js').AccountDID} customer - * @param {string} resourceCID + * @param {import('../types.js').UnknownLink} resource * @param {number} bytes * @param {Date} servedAt */ - async record(customer, resourceCID, bytes, servedAt) { + async record(customer, resource, bytes, servedAt) { const egressData = { customer, - resourceCID, + resource, bytes, servedAt: servedAt.toISOString(), } diff --git a/packages/w3up-client/src/capability/usage.js b/packages/w3up-client/src/capability/usage.js index fb31ab86d..599813b40 100644 --- a/packages/w3up-client/src/capability/usage.js +++ b/packages/w3up-client/src/capability/usage.js @@ -48,7 +48,7 @@ export class UsageClient extends Base { { agent: this.agent }, { ...options, space, egressData } ) - /* c8 ignore next 7 */ + /* c8 ignore next 5 */ if (!out.ok) { throw new Error(`failed ${UsageCapabilities.record.can} invocation`, { cause: out.error, @@ -109,7 +109,7 @@ export const record = async ( nonce, nb: { customer: egressData.customer, - resourceCID: egressData.resourceCID, + resource: egressData.resource, bytes: egressData.bytes, servedAt: Math.floor(new Date(egressData.servedAt).getTime() / 1000), }, diff --git a/packages/w3up-client/test/capability/usage.test.js b/packages/w3up-client/test/capability/usage.test.js index 5a998dec5..d396831dd 100644 --- a/packages/w3up-client/test/capability/usage.test.js +++ b/packages/w3up-client/test/capability/usage.test.js @@ -95,15 +95,15 @@ export const UsageClient = Test.withContext({ }) const car = await randomCAR(128) - const resourceCID = car.cid - await alice.capability.upload.add(car.roots[0], [resourceCID]) + const resource = car.cid + await alice.capability.upload.add(car.roots[0], [resource]) const result = await alice.capability.upload.get(car.roots[0]) assert.ok(result) const record = await alice.capability.usage.record(space.did(), { customer: 'did:mailto:alice@web.mail', - resourceCID: resourceCID.link().toString(), + resource: resource.link(), bytes: car.size, servedAt: new Date().toISOString(), }) From 4f48123a7e1051def7fb3ff52e35d068aca84aad Mon Sep 17 00:00:00 2001 From: Felipe Forbeck Date: Tue, 22 Oct 2024 10:41:25 -0300 Subject: [PATCH 3/3] fix: find customer by provider and consumer --- packages/capabilities/src/types.ts | 1 + packages/upload-api/src/usage/record.js | 20 +++++++++++++++++-- .../test/storage/provisions-storage.js | 1 + 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 66c5794d4..da3426c6b 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -208,6 +208,7 @@ export interface ConsumerGetSuccess { allocated: number limit: number subscription: string + customer: AccountDID } export interface ConsumerNotFound extends Ucanto.Failure { name: 'ConsumerNotFound' diff --git a/packages/upload-api/src/usage/record.js b/packages/upload-api/src/usage/record.js index b99955f07..e4da45161 100644 --- a/packages/upload-api/src/usage/record.js +++ b/packages/upload-api/src/usage/record.js @@ -11,9 +11,25 @@ export const provide = (context) => * @param {API.UsageServiceContext} context * @returns {Promise>} */ -const record = async ({ capability }, context) => { +const record = async ({ capability, invocation }, context) => { + const provider = /** @type {`did:web:${string}`} */ ( + invocation.audience.did() + ) + const consumerResponse = await context.provisionsStorage.getConsumer( + provider, + capability.with + ) + if (consumerResponse.error) { + return { + error: { + name: 'EgressRecordFailure', + message: `Failed to get consumer`, + }, + } + } + const consumer = consumerResponse.ok const res = await context.usageStorage.record( - capability.nb.customer, + consumer.customer, capability.nb.resource, capability.nb.bytes, new Date(capability.nb.servedAt * 1000) diff --git a/packages/upload-api/test/storage/provisions-storage.js b/packages/upload-api/test/storage/provisions-storage.js index 272c44a7a..78f38426e 100644 --- a/packages/upload-api/test/storage/provisions-storage.js +++ b/packages/upload-api/test/storage/provisions-storage.js @@ -152,6 +152,7 @@ export class ProvisionsStorage { allocated: 0, limit: 100, subscription: itemKey(provision), + customer: provision.customer, }, } } else {