fix: add backup url parts to list (#2675)

Uses same approach and naming (parts) as web3.storage changes:
web3-storage/web3.storage#2347

Closes #2674

packages/api/package.json

@@ -21,9 +21,9 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.37.0",
     "@cfworker/json-schema": "^1.8.3",
-    "@ipld/car": "^3.1.20",
-    "@ipld/dag-cbor": "^6.0.13",
-    "@ipld/dag-pb": "^2.1.16",
+    "@ipld/car": "^5.3.1",
+    "@ipld/dag-cbor": "^9.2.0",
+    "@ipld/dag-pb": "^4.1.1",
     "@magic-sdk/admin": "1.4.0",
     "@nftstorage/ipfs-cluster": "^5.0.1",
     "@noble/ed25519": "^1.6.1",
@@ -41,7 +41,7 @@
     "it-last": "^2.0.0",
     "linkdex": "^3.0.0",
     "merge-options": "^3.0.4",
-    "multiformats": "^9.6.4",
+    "multiformats": "^13.1.1",
     "nanoid": "^3.1.30",
     "one-webcrypto": "^1.0.3",
     "p-retry": "^5.1.1",

packages/api/src/bindings.d.ts

@@ -206,6 +206,11 @@ export type NFT = {
    * Date this NFT was created in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) format: YYYY-MM-DDTHH:MM:SSZ.
    */
   created: string
+  /**
+   * the graph from `cid` can be recreated from the blocks in these parts
+   * @see https://github.com/web3-storage/content-claims#partition-claim
+   */
+  parts: string[]
 }
 
 export type NFTResponse = NFT & {

packages/api/src/routes/nfts-store.js

@@ -111,6 +111,7 @@ async function cborEncode(value, bs) {
   const bytes = CBOR.encode(value)
   const digest = await sha256.digest(bytes)
   const cid = CID.createV1(CBOR.code, digest)
+  // @ts-expect-error different CID versions
   await bs.put(cid, bytes)
   return cid
 }
@@ -160,6 +161,7 @@ async function unixFsEncodeDir(files, bs) {
     const content = new Uint8Array(await f.arrayBuffer())
     input.push({ path: f.name, content })
   }
+  // @ts-expect-error different CID versions
   return unixFsEncode(input, bs, {
     wrapWithDirectory: true,
   })
@@ -174,6 +176,7 @@ async function unixFsEncodeDir(files, bs) {
 async function unixFsEncodeString(str, bs) {
   const content = new TextEncoder().encode(str)
   const ic = { path: '', content }
+  // @ts-expect-error different CID versions
   return unixFsEncode(ic, bs, {
     wrapWithDirectory: false,
   })

packages/api/src/utils/car.js

@@ -5,7 +5,7 @@ import { CarIndexer } from '@ipld/car/indexer'
 import { MultihashIndexSortedWriter } from 'cardex'
 
 /**
- * @typedef {import('multiformats/block').Block<unknown>} Block
+ * @typedef {import('multiformats/block').Block<unknown, number, number, 1>} Block
  */
 
 /**

packages/api/src/utils/db-transforms.js

@@ -1,4 +1,6 @@
-import * as cluster from '../cluster.js'
+import * as Link from 'multiformats/link'
+import * as Digest from 'multiformats/hashes/digest'
+import { fromString } from 'uint8arrays'
 
 /**
  * We mixed upload type and content type. This was split into `type` and
@@ -20,6 +22,13 @@ const typeMap = {
  * @param {string} [sourceCid] - User input CID so we can return the same cid version back
  */
 export function toNFTResponse(upload, sourceCid) {
+  // get hash links to CARs that contain parts of this upload
+  /** @type {string[]} */
+  const parts = [
+    // from upload table 'backup_urls' column
+    ...carCidV1Base32sFromBackupUrls(upload.backup_urls ?? []),
+  ]
+
   /** @type {import('../bindings').NFTResponse} */
   const nft = {
     cid: sourceCid || upload.source_cid,
@@ -29,6 +38,7 @@ export function toNFTResponse(upload, sourceCid) {
     files: upload.files,
     size: upload.content.dag_size || 0,
     name: upload.name,
+    parts,
     pin: {
       cid: sourceCid || upload.source_cid,
       created: upload.content.pin[0].inserted_at,
@@ -101,3 +111,53 @@ function transformPinStatus(status) {
       return 'failed'
   }
 }
+
+/**
+ * given array of backup_urls from uploads table, return a corresponding set of CAR CIDv1 using base32 multihash
+ * for any CAR files in the backup_urls.
+ *
+ * @param {unknown[]} backupUrls
+ * @returns {Iterable<string>}
+ */
+function carCidV1Base32sFromBackupUrls(backupUrls) {
+  const carCidStrings = new Set()
+  for (const backupUrl of backupUrls) {
+    let carCid
+    try {
+      // @ts-expect-error database exported types assumes unknown
+      carCid = bucketKeyToPartCID(backupUrl)
+    } catch (error) {
+      console.warn('error extracting car CID from bucket URL', error)
+    }
+    if (!carCid) continue
+    carCidStrings.add(carCid.toString())
+  }
+  return carCidStrings
+}
+
+const CAR_CODE = 0x0202
+
+/**
+ * Attempts to extract a CAR CID from a bucket key.
+ *
+ * @param {string} key
+ */
+const bucketKeyToPartCID = (key) => {
+  const filename = String(key.split('/').at(-1))
+  const [hash] = filename.split('.')
+  try {
+    // recent buckets encode CAR CID in filename
+    const cid = Link.parse(hash).toV1()
+    if (cid.code === CAR_CODE) return cid
+    throw new Error('not a CAR CID')
+  } catch (err) {
+    // older buckets base32 encode a CAR multihash <base32(car-multihash)>.car
+    try {
+      const digestBytes = fromString(hash, 'base32')
+      const digest = Digest.decode(digestBytes)
+      return Link.create(CAR_CODE, digest)
+    } catch (error) {
+      // console.warn('error trying to create CID from s3 key', error)
+    }
+  }
+}

packages/api/src/utils/linkdex.js

@@ -58,7 +58,6 @@ export class LinkdexApi {
         if (!res || !res.body) throw new Error(`failed to get CAR: ${cid}`)
         const carBlocks = await CarBlockIterator.fromIterable(res.body)
         for await (const block of carBlocks) {
-          // @ts-expect-error block types not match up
           index.decodeAndIndex(block)
         }
       })

packages/api/test/nfts-list.spec.js

@@ -182,3 +182,69 @@ test.serial('should list only active nfts', async (t) => {
   t.is(value.length, 1)
   t.is(value[0].cid, cidv1)
 })
+
+test.serial('should list nfts with their parts', async (t) => {
+  const client = await createClientWithUser(t)
+  const mf = getMiniflareContext(t)
+  const cidv1 = 'bafybeiaj5yqocsg5cxsuhtvclnh4ulmrgsmnfbhbrfxrc3u2kkh35mts4e'
+  const cidv0 = 'QmP1QyqiRtQLbGBr5hLVX7NCmrJmJbGdp45x6DnPssMB9i'
+  const exampleCarParkUrl =
+    'https://carpark-dev.web3.storage/bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea/bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea.car'
+  const exampleS3Url = `https://dotstorage-dev-0.s3.us-east-1.amazonaws.com/raw/${cidv1}/2/ciqplrl7tuebgpzbo5nqlqus5hj2kowxzz7ayr4z6ao2ftg7ibcr3ca.car`
+  await client.client.createUpload({
+    content_cid: cidv1,
+    source_cid: cidv0,
+    type: 'Blob',
+    user_id: client.userId,
+    dag_size: 100,
+    backup_urls: [new URL(exampleCarParkUrl), new URL(exampleS3Url)],
+  })
+
+  const res = await mf.dispatchFetch('http://miniflare.test', {
+    headers: { Authorization: `Bearer ${client.token}` },
+  })
+  const { ok, value } = await res.json()
+
+  t.true(ok)
+  t.is(value.length, 1)
+  t.is(value[0].cid, cidv0)
+  t.truthy(Array.isArray(value[0].parts), 'upload.parts is an array')
+  t.deepEqual(value[0].parts, [
+    // this corresponds to `exampleCarParkUrl`
+    'bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea',
+  ])
+})
+
+test.serial(
+  'should list nfts with their parts including w3s.link',
+  async (t) => {
+    const client = await createClientWithUser(t)
+    const mf = getMiniflareContext(t)
+    const cidv1 = 'bafybeiaj5yqocsg5cxsuhtvclnh4ulmrgsmnfbhbrfxrc3u2kkh35mts4e'
+    const cidv0 = 'QmP1QyqiRtQLbGBr5hLVX7NCmrJmJbGdp45x6DnPssMB9i'
+    const exampleS3Url = `https://dotstorage-dev-0.s3.us-east-1.amazonaws.com/raw/${cidv1}/2/ciqplrl7tuebgpzbo5nqlqus5hj2kowxzz7ayr4z6ao2ftg7ibcr3ca.car`
+    const exampleW3sUrl = `https://w3s.link/ipfs/bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea`
+    await client.client.createUpload({
+      content_cid: cidv1,
+      source_cid: cidv0,
+      type: 'Blob',
+      user_id: client.userId,
+      dag_size: 100,
+      backup_urls: [new URL(exampleW3sUrl), new URL(exampleS3Url)],
+    })
+
+    const res = await mf.dispatchFetch('http://miniflare.test', {
+      headers: { Authorization: `Bearer ${client.token}` },
+    })
+    const { ok, value } = await res.json()
+
+    t.true(ok)
+    t.is(value.length, 1)
+    t.is(value[0].cid, cidv0)
+    t.truthy(Array.isArray(value[0].parts), 'upload.parts is an array')
+    t.deepEqual(value[0].parts, [
+      // this corresponds to `exampleW3sUrl`
+      'bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea',
+    ])
+  }
+)

packages/api/test/scripts/car.js

@@ -14,6 +14,7 @@ export async function createCar(str) {
     hasher: sha256,
   })
   const root = block.cid
+  // @ts-expect-error different CID versions
   const car = await CAR.encode([root], [block])
   return { root, car }
 }

yarn.lock

@@ -2625,7 +2625,7 @@
   resolved "https://registry.yarnpkg.com/@inquirer/type/-/type-1.2.1.tgz#fbc7ab3a2e5050d0c150642d5e8f5e88faa066b8"
   integrity sha512-xwMfkPAxeo8Ji/IxfUSqzRi0/+F2GIqJmpc5/thelgMGsjNZcjDDRBO9TLXT1s/hdx/mK5QbVIvgoLIFgXhTMQ==
 
-"@ipld/car@^3.0.1", "@ipld/car@^3.1.20", "@ipld/car@^3.2.3":
+"@ipld/car@^3.0.1", "@ipld/car@^3.2.3":
   version "3.2.4"
   resolved "https://registry.yarnpkg.com/@ipld/car/-/car-3.2.4.tgz#115951ba2255ec51d865773a074e422c169fb01c"
   integrity sha512-rezKd+jk8AsTGOoJKqzfjLJ3WVft7NZNH95f0pfPbicROvzTyvHCNy567HzSUd6gRXZ9im29z5ZEv9Hw49jSYw==
@@ -2654,6 +2654,16 @@
     multiformats "^13.0.0"
     varint "^6.0.0"
 
+"@ipld/car@^5.3.1":
+  version "5.3.1"
+  resolved "https://registry.yarnpkg.com/@ipld/car/-/car-5.3.1.tgz#6a967b2f929cab007466edab3171c18f489036d4"
+  integrity sha512-8fNkYAZvL9yX2zesF32k7tYqUDGG41felmmBnwjCZJto06QXCb0NOMPJc/mhNgnVa5gkKqxPO1ZdSoHuaYcVSw==
+  dependencies:
+    "@ipld/dag-cbor" "^9.0.7"
+    cborg "^4.0.5"
+    multiformats "^13.0.0"
+    varint "^6.0.0"
+
 "@ipld/dag-cbor@^6.0.13", "@ipld/dag-cbor@^6.0.3":
   version "6.0.15"
   resolved "https://registry.yarnpkg.com/@ipld/dag-cbor/-/dag-cbor-6.0.15.tgz#aebe7a26c391cae98c32faedb681b1519e3d2372"
@@ -2670,7 +2680,7 @@
     cborg "^1.6.0"
     multiformats "^9.5.4"
 
-"@ipld/dag-cbor@^9.0.0", "@ipld/dag-cbor@^9.0.3", "@ipld/dag-cbor@^9.0.5", "@ipld/dag-cbor@^9.0.6", "@ipld/dag-cbor@^9.0.7":
+"@ipld/dag-cbor@^9.0.0", "@ipld/dag-cbor@^9.0.3", "@ipld/dag-cbor@^9.0.5", "@ipld/dag-cbor@^9.0.6", "@ipld/dag-cbor@^9.0.7", "@ipld/dag-cbor@^9.2.0":
   version "9.2.0"
   resolved "https://registry.yarnpkg.com/@ipld/dag-cbor/-/dag-cbor-9.2.0.tgz#3a3f0bee02d7e1c2f15582e896843d5b00fbba9f"
   integrity sha512-N14oMy0q4gM6OuZkIpisKe0JBSjf1Jb39VI+7jMLiWX9124u1Z3Fdj/Tag1NA0cVxxqWDh0CqsjcVfOKtelPDA==
@@ -2694,7 +2704,7 @@
     cborg "^1.5.4"
     multiformats "^9.5.4"
 
-"@ipld/dag-pb@^2.0.2", "@ipld/dag-pb@^2.1.16":
+"@ipld/dag-pb@^2.0.2":
   version "2.1.18"
   resolved "https://registry.yarnpkg.com/@ipld/dag-pb/-/dag-pb-2.1.18.tgz#12d63e21580e87c75fd1a2c62e375a78e355c16f"
   integrity sha512-ZBnf2fuX9y3KccADURG5vb9FaOeMjFkCrNysB0PtftME/4iCTjxfaLoNq/IAh5fTqUOMXvryN6Jyka4ZGuMLIg==
@@ -2708,6 +2718,13 @@
   dependencies:
     multiformats "^13.1.0"
 
+"@ipld/dag-pb@^4.1.1":
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/@ipld/dag-pb/-/dag-pb-4.1.1.tgz#fb5c253ad0f2ced00832e19b7c58985861a7fa34"
+  integrity sha512-wsSNjIvcABXuH9MKXpvRGMXsS20+Kf2Q0Hq2+2dxN6Wpw/K0kDF3nDmCnO6wlpninQ0vzx1zq54O3ttn5pTH9A==
+  dependencies:
+    multiformats "^13.1.0"
+
 "@ipld/dag-ucan@^3.4.0":
   version "3.4.0"
   resolved "https://registry.yarnpkg.com/@ipld/dag-ucan/-/dag-ucan-3.4.0.tgz#bc955fb6506cff6a0d876476d06ca98ec8b15b4d"
@@ -16170,6 +16187,11 @@ multiformats@^13.0.0, multiformats@^13.1.0:
   resolved "https://registry.yarnpkg.com/multiformats/-/multiformats-13.1.0.tgz#5aa9d2175108a448fc3bdb54ba8a3d0b6cab3ac3"
   integrity sha512-HzdtdBwxsIkzpeXzhQ5mAhhuxcHbjEHH+JQoxt7hG/2HGFjjwyolLo7hbaexcnhoEuV4e0TNJ8kkpMjiEYY4VQ==
 
+multiformats@^13.1.1:
+  version "13.1.1"
+  resolved "https://registry.yarnpkg.com/multiformats/-/multiformats-13.1.1.tgz#b22ce4df26330d2cf0d69f5bdcbc9a787095a6e5"
+  integrity sha512-JiptvwMmlxlzIlLLwhCi/srf/nk409UL0eUBr0kioRJq15hqqKyg68iftrBvhCRjR6Rw4fkNnSc4ZJXJDuta/Q==
+
 multiformats@^9.0.0, multiformats@^9.0.4, multiformats@^9.4.13, multiformats@^9.4.2, multiformats@^9.4.5, multiformats@^9.4.7, multiformats@^9.5.4, multiformats@^9.6.3, multiformats@^9.6.4:
   version "9.7.1"
   resolved "https://registry.yarnpkg.com/multiformats/-/multiformats-9.7.1.tgz#ab348e5fd6f8e7fb3fd56033211bda48854e2173"
@@ -20623,7 +20645,7 @@ string-argv@^0.3.1:
   resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.1.tgz#95e2fbec0427ae19184935f816d74aaa4c5c19da"
   integrity sha512-a1uQGz7IyVy9YwhqjZIZu1c8JO8dNIe20xBmSS6qu9kv++k3JGzCVmprbNN5Kn+BgzD5E7YYwg1CcjuJMRNsvg==
 
-"string-width-cjs@npm:string-width@^4.2.0":
+"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -20641,15 +20663,6 @@ string-width@^1.0.1:
     is-fullwidth-code-point "^1.0.0"
     strip-ansi "^3.0.0"
 
-"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
-  version "4.2.3"
-  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
-  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
-  dependencies:
-    emoji-regex "^8.0.0"
-    is-fullwidth-code-point "^3.0.0"
-    strip-ansi "^6.0.1"
-
 string-width@^5.0.0, string-width@^5.0.1, string-width@^5.1.2:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-5.1.2.tgz#14f8daec6d81e7221d2a357e668cab73bdbca794"
@@ -20791,7 +20804,7 @@ stringify-entities@^4.0.0:
     character-entities-html4 "^2.0.0"
     character-entities-legacy "^3.0.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -20805,13 +20818,6 @@ strip-ansi@^3.0.0, strip-ansi@^3.0.1:
   dependencies:
     ansi-regex "^2.0.0"
 
-strip-ansi@^6.0.0, strip-ansi@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
-  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
-  dependencies:
-    ansi-regex "^5.0.1"
-
 strip-ansi@^7.0.0, strip-ansi@^7.0.1:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2"
@@ -22893,7 +22899,7 @@ wrangler@^2.0.23:
   optionalDependencies:
     fsevents "~2.3.2"
 
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
   integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@@ -22911,15 +22917,6 @@ wrap-ansi@^6.2.0:
     string-width "^4.1.0"
     strip-ansi "^6.0.0"
 
-wrap-ansi@^7.0.0:
-  version "7.0.0"
-  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
-  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
-  dependencies:
-    ansi-styles "^4.0.0"
-    string-width "^4.1.0"
-    strip-ansi "^6.0.0"
-
 wrap-ansi@^8.1.0:
   version "8.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"