refactor: switch to rust piece CID calculator (#930)

Currently blocked on top level await issue. --------- Co-authored-by: Irakli Gozalishvili <[email protected]>
storacha · Sep 20, 2023 · cf7e203 · cf7e203
1 parent 329354d
commit cf7e203
Show file tree

Hide file tree

Showing 9 changed files with 816 additions and 462 deletions.
diff --git a/packages/access-client/package.json b/packages/access-client/package.json
@@ -96,7 +96,7 @@
     "assert": "^2.0.0",
     "hd-scripts": "^4.0.0",
     "mocha": "^10.2.0",
-    "playwright-test": "^8.1.2",
+    "playwright-test": "^12.3.4",
     "sade": "^1.8.1",
     "sinon": "^15.0.3",
     "typescript": "4.9.5",

diff --git a/packages/capabilities/package.json b/packages/capabilities/package.json
@@ -69,7 +69,7 @@
     "assert": "^2.0.0",
     "hd-scripts": "^4.0.0",
     "mocha": "^10.2.0",
-    "playwright-test": "^8.1.2",
+    "playwright-test": "^12.3.4",
     "type-fest": "^3.3.0",
     "typescript": "4.9.5",
     "watch": "^1.0.2"

diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts
@@ -16,7 +16,7 @@ import * as RateLimitCaps from './rate-limit.js'
 import * as FilecoinCaps from './filecoin.js'
 import * as AdminCaps from './admin.js'
 
-export type { Unit }
+export type { Unit, PieceLink }
 
 export type AccountDID = DID<'mailto'>
 export type SpaceDID = DID<'key'>

diff --git a/packages/filecoin-client/package.json b/packages/filecoin-client/package.json
@@ -55,8 +55,8 @@
     "dist/src/**/*.d.ts.map"
   ],
   "dependencies": {
-    "@ipld/dag-ucan": "^3.3.2",
     "@ipld/dag-cbor": "^9.0.0",
+    "@ipld/dag-ucan": "^3.3.2",
     "@ucanto/client": "^8.0.0",
     "@ucanto/core": "^8.0.0",
     "@ucanto/interface": "^8.0.0",
@@ -77,7 +77,7 @@
     "mocha": "^10.2.0",
     "multiformats": "^11.0.2",
     "npm-run-all": "^4.1.5",
-    "playwright-test": "^8.1.2",
+    "playwright-test": "^12.3.4",
     "typescript": "4.9.5"
   },
   "eslintConfig": {

diff --git a/packages/upload-client/package.json b/packages/upload-client/package.json
@@ -72,7 +72,7 @@
     "@ucanto/interface": "^8.0.0",
     "@ucanto/transport": "^8.0.0",
     "@web3-storage/capabilities": "workspace:^",
-    "@web3-storage/data-segment": "^3.0.1",
+    "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "^1.1.4",
     "ipfs-utils": "^9.0.14",
     "multiformats": "^11.0.2",
     "p-retry": "^5.1.2",
@@ -93,7 +93,7 @@
     "ipfs-unixfs-exporter": "^10.0.0",
     "mocha": "^10.2.0",
     "npm-run-all": "^4.1.5",
-    "playwright-test": "^8.1.2",
+    "playwright-test": "^12.3.4",
     "typescript": "4.9.5"
   },
   "eslintConfig": {

diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js
@@ -1,5 +1,8 @@
 import { Parallel } from 'parallel-transform-web'
-import { Piece } from '@web3-storage/data-segment'
+import * as PieceHasher from 'fr32-sha2-256-trunc254-padded-binary-tree-multihash'
+import * as Link from 'multiformats/link'
+import * as Digest from 'multiformats/hashes/digest'
+import * as raw from 'multiformats/codecs/raw'
 import * as Store from './store.js'
 import * as Upload from './upload.js'
 import * as UnixFS from './unixfs.js'
@@ -10,6 +13,7 @@ export { Store, Upload, UnixFS, CAR }
 export * from './sharding.js'
 
 const CONCURRENT_REQUESTS = 3
+const PIECE_MULTIHASH_SIZE = PieceHasher.prefix.length + PieceHasher.size
 
 /**
  * Uploads a file to the service and returns the root data CID for the
@@ -124,10 +128,19 @@ async function uploadBlockStream(conf, blocks, options = {}) {
         const bytes = new Uint8Array(await car.arrayBuffer())
         const [cid, piece] = await Promise.all([
           Store.add(conf, bytes, options),
-          Piece.fromPayload(bytes),
+          (async () => {
+            const hasher = PieceHasher.create()
+            const digestBytes = new Uint8Array(PIECE_MULTIHASH_SIZE)
+            hasher.write(bytes)
+            hasher.digestInto(digestBytes, 0, true)
+            const digest = Digest.decode(digestBytes)
+            return /** @type {import('@web3-storage/capabilities/types').PieceLink} */ (
+              Link.create(raw.code, digest)
+            )
+          })(),
         ])
         const { version, roots, size } = car
-        return { version, roots, size, cid, piece: piece.link }
+        return { version, roots, size, cid, piece }
       })
     )
     .pipeTo(

diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts
@@ -22,9 +22,9 @@ import {
   UploadAdd,
   UploadList,
   UploadRemove,
+  PieceLink,
 } from '@web3-storage/capabilities/types'
 import * as UnixFS from '@ipld/unixfs/src/unixfs'
-import type { PieceLink } from '@web3-storage/data-segment'
 
 export type { PieceLink }
 

diff --git a/packages/w3up-client/package.json b/packages/w3up-client/package.json
@@ -96,7 +96,7 @@
     "mocha": "^10.1.0",
     "multiformats": "^11.0.0",
     "npm-run-all": "^4.1.5",
-    "playwright-test": "^8.1.1",
+    "playwright-test": "^12.3.4",
     "standard": "^17.0.0",
     "typedoc": "^0.23.24",
     "typedoc-plugin-markdown": "^3.14.0",