-
Notifications
You must be signed in to change notification settings - Fork 21
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: usage/record capability definition #1562
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -131,6 +131,10 @@ export type UsageReport = InferInvokedCapability<typeof UsageCaps.report> | |
export type UsageReportSuccess = Record<ProviderDID, UsageData> | ||
export type UsageReportFailure = Ucanto.Failure | ||
|
||
export type EgressRecord = InferInvokedCapability<typeof UsageCaps.record> | ||
export type EgressRecordSuccess = Unit | ||
export type EgressRecordFailure = Ucanto.Failure | ||
|
||
export interface UsageData { | ||
/** Provider the report concerns, e.g. `did:web:web3.storage` */ | ||
provider: ProviderDID | ||
|
@@ -161,6 +165,17 @@ export interface UsageData { | |
}> | ||
} | ||
|
||
export interface EgressData { | ||
/** Id of the customer that is being billed. */ | ||
customer: AccountDID | ||
/** CID of the resource that was served. */ | ||
resource: UnknownLink | ||
/** Amount of bytes served. */ | ||
bytes: number | ||
/** ISO datetime that the bytes were served at. */ | ||
servedAt: ISO8601Date | ||
} | ||
|
||
// Provider | ||
export type ProviderAdd = InferInvokedCapability<typeof provider.add> | ||
// eslint-disable-next-line @typescript-eslint/no-empty-interface | ||
|
@@ -193,6 +208,7 @@ export interface ConsumerGetSuccess { | |
allocated: number | ||
limit: number | ||
subscription: string | ||
customer: AccountDID | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
} | ||
export interface ConsumerNotFound extends Ucanto.Failure { | ||
name: 'ConsumerNotFound' | ||
|
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
@@ -1,5 +1,5 @@ | ||||||
import { capability, ok, Schema } from '@ucanto/validator' | ||||||
import { and, equal, equalWith, SpaceDID } from './utils.js' | ||||||
import { AccountDID, and, equal, equalWith, SpaceDID } from './utils.js' | ||||||
|
||||||
/** | ||||||
* Capability can only be delegated (but not invoked) allowing audience to | ||||||
|
@@ -40,3 +40,22 @@ export const report = capability({ | |||||
) | ||||||
}, | ||||||
}) | ||||||
|
||||||
/** | ||||||
* Capability can be invoked by an agent to record usage data for a given resource. | ||||||
*/ | ||||||
export const record = capability({ | ||||||
can: 'usage/record', | ||||||
with: SpaceDID, | ||||||
nb: Schema.struct({ | ||||||
/** MailTo DID of the customer that is being billed. */ | ||||||
customer: AccountDID, | ||||||
fforbeck marked this conversation as resolved.
Show resolved
Hide resolved
Comment on lines
+51
to
+52
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
/** CID of the resource that was served. */ | ||||||
resource: Schema.link(), | ||||||
/** Amount of bytes served. */ | ||||||
bytes: Schema.integer().greaterThan(0), | ||||||
/** Timestamp of the event in seconds after Unix epoch. */ | ||||||
servedAt: Schema.integer().greaterThan(-1), | ||||||
}), | ||||||
derives: equalWith, | ||||||
}) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,8 +1,10 @@ | ||
import { Failure, Result } from '@ucanto/interface' | ||
import { Failure, Result, UnknownLink } from '@ucanto/interface' | ||
import { | ||
ProviderDID, | ||
SpaceDID, | ||
UsageData, | ||
EgressData, | ||
AccountDID, | ||
} from '@web3-storage/capabilities/types' | ||
|
||
export type { UsageData } | ||
|
@@ -13,4 +15,10 @@ export interface UsageStorage { | |
space: SpaceDID, | ||
period: { from: Date; to: Date } | ||
) => Promise<Result<UsageData, Failure>> | ||
record: ( | ||
customer: AccountDID, | ||
resource: UnknownLink, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Record space DID also? Would be good to be able to enumerate egress usage by space. |
||
bytes: number, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Would be good to get a comment here - what is the resource? It's the root CID of a DAG (not the CID of a CAR) or I imagine it could also be literally a CID of any sub-DAG within a DAG. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It is the same data CID we use in the gateway to fetch the content. It is pulled from the URL via There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we have enough information to identify the blob(s)/shard(s) the resource was served from? Feels like a useful thing to track here, especially if the CID being used is not a root CID, but a sub-DAG. I'm thinking about users querying why a certain CID has used so much egress but it is not one of the root CIDs from |
||
servedAt: Date | ||
) => Promise<Result<EgressData, Failure>> | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We should add a |
||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,8 @@ | ||
import { provide } from './usage/report.js' | ||
import { provide as provideReport } from './usage/report.js' | ||
import { provide as provideRecord } from './usage/record.js' | ||
|
||
/** @param {import('./types.js').UsageServiceContext} context */ | ||
export const createService = (context) => ({ report: provide(context) }) | ||
export const createService = (context) => ({ | ||
report: provideReport(context), | ||
record: provideRecord(context), | ||
}) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
import * as API from '../types.js' | ||
import * as Provider from '@ucanto/server' | ||
import { Usage } from '@web3-storage/capabilities' | ||
|
||
/** @param {API.UsageServiceContext} context */ | ||
export const provide = (context) => | ||
Provider.provide(Usage.record, (input) => record(input, context)) | ||
|
||
/** | ||
* @param {API.Input<Usage.record>} input | ||
* @param {API.UsageServiceContext} context | ||
* @returns {Promise<API.Result<API.EgressRecordSuccess, API.EgressRecordFailure>>} | ||
*/ | ||
const record = async ({ capability, invocation }, context) => { | ||
const provider = /** @type {`did:web:${string}`} */ ( | ||
invocation.audience.did() | ||
) | ||
const consumerResponse = await context.provisionsStorage.getConsumer( | ||
provider, | ||
capability.with | ||
) | ||
if (consumerResponse.error) { | ||
return { | ||
fforbeck marked this conversation as resolved.
Show resolved
Hide resolved
|
||
error: { | ||
name: 'EgressRecordFailure', | ||
message: `Failed to get consumer`, | ||
fforbeck marked this conversation as resolved.
Show resolved
Hide resolved
|
||
}, | ||
} | ||
} | ||
const consumer = consumerResponse.ok | ||
const res = await context.usageStorage.record( | ||
fforbeck marked this conversation as resolved.
Show resolved
Hide resolved
|
||
consumer.customer, | ||
capability.nb.resource, | ||
capability.nb.bytes, | ||
new Date(capability.nb.servedAt * 1000) | ||
Comment on lines
+15
to
+35
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I updated the function to find the |
||
) | ||
if (res.error) return res | ||
|
||
return res | ||
fforbeck marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} |
Original file line number | Diff line number | Diff line change | ||
---|---|---|---|---|
|
@@ -31,6 +31,32 @@ export class UsageClient extends Base { | |||
|
||||
return out.ok | ||||
} | ||||
|
||||
/** | ||||
* Record egress data for the customer and served resource. | ||||
* | ||||
* Required delegated capabilities: | ||||
* - `usage/record` | ||||
* | ||||
* @param {import('../types.js').SpaceDID} space | ||||
* @param {API.EgressData} egressData | ||||
* @param {object} [options] | ||||
* @param {string} [options.nonce] | ||||
*/ | ||||
async record(space, egressData, options) { | ||||
const out = await record( | ||||
{ agent: this.agent }, | ||||
{ ...options, space, egressData } | ||||
) | ||||
/* c8 ignore next 5 */ | ||||
fforbeck marked this conversation as resolved.
Show resolved
Hide resolved
|
||||
if (!out.ok) { | ||||
throw new Error(`failed ${UsageCapabilities.record.can} invocation`, { | ||||
cause: out.error, | ||||
}) | ||||
} | ||||
|
||||
return out.ok | ||||
} | ||||
} | ||||
|
||||
/** | ||||
|
@@ -61,3 +87,32 @@ export const report = async ( | |||
}) | ||||
return receipt.out | ||||
} | ||||
|
||||
/** | ||||
* Record egress data for the customer and served resource. | ||||
* | ||||
* @param {{agent: API.Agent}} client | ||||
* @param {object} options | ||||
* @param {API.SpaceDID} options.space | ||||
* @param {API.EgressData} options.egressData - | ||||
* @param {string} [options.nonce] | ||||
* @param {API.Delegation[]} [options.proofs] | ||||
* @returns {Promise<API.Result<API.Unit, API.EgressRecordFailure>>} | ||||
*/ | ||||
export const record = async ( | ||||
{ agent }, | ||||
{ space, egressData, nonce, proofs = [] } | ||||
) => { | ||||
const receipt = await agent.invokeAndExecute(UsageCapabilities.record, { | ||||
with: space, | ||||
proofs, | ||||
nonce, | ||||
nb: { | ||||
customer: egressData.customer, | ||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||
resource: egressData.resource, | ||||
bytes: egressData.bytes, | ||||
servedAt: Math.floor(new Date(egressData.servedAt).getTime() / 1000), | ||||
}, | ||||
}) | ||||
return receipt.out | ||||
} |
Original file line number | Diff line number | Diff line change | ||
---|---|---|---|---|
|
@@ -2,6 +2,7 @@ import { AgentData } from '@web3-storage/access/agent' | |||
import { Client } from '../../src/client.js' | ||||
import * as Test from '../test.js' | ||||
import { receiptsEndpoint } from '../helpers/utils.js' | ||||
import { randomCAR } from '../helpers/random.js' | ||||
|
||||
export const UsageClient = Test.withContext({ | ||||
report: { | ||||
|
@@ -68,6 +69,47 @@ export const UsageClient = Test.withContext({ | |||
assert.deepEqual(report, {}) | ||||
}, | ||||
}, | ||||
record: { | ||||
'should record egress': async ( | ||||
assert, | ||||
{ connection, provisionsStorage } | ||||
) => { | ||||
const alice = new Client(await AgentData.create(), { | ||||
// @ts-ignore | ||||
serviceConf: { | ||||
access: connection, | ||||
upload: connection, | ||||
}, | ||||
}) | ||||
|
||||
const space = await alice.createSpace('test') | ||||
const auth = await space.createAuthorization(alice) | ||||
await alice.addSpace(auth) | ||||
|
||||
// Then we setup a billing for this account | ||||
await provisionsStorage.put({ | ||||
// @ts-expect-error | ||||
provider: connection.id.did(), | ||||
account: alice.agent.did(), | ||||
consumer: space.did(), | ||||
}) | ||||
|
||||
const car = await randomCAR(128) | ||||
const resource = car.cid | ||||
await alice.capability.upload.add(car.roots[0], [resource]) | ||||
|
||||
const result = await alice.capability.upload.get(car.roots[0]) | ||||
assert.ok(result) | ||||
|
||||
const record = await alice.capability.usage.record(space.did(), { | ||||
customer: 'did:mailto:[email protected]', | ||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||
resource: resource.link(), | ||||
bytes: car.size, | ||||
servedAt: new Date().toISOString(), | ||||
}) | ||||
assert.ok(record) | ||||
}, | ||||
}, | ||||
}) | ||||
|
||||
Test.test({ UsageClient }) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The service needs to obtain this.