fix: grow routing table trie towards node kad-id

packages/kad-dht/README.md

@@ -102,6 +102,45 @@ const peerInfo = await node.peerRouting.findPeer(peerId)
 console.info(peerInfo) // peer id, multiaddrs
 ```
 
+## The routing table
+
+This module uses a binary trie for it's routing table. By default the trie
+will only grow in the direction of the KadID of the current peer:
+
+```
+Peer KadID: 01101...
+
+            InternalBucket
+                0 / \ 1
+    InternalBucket   LeafBucket
+        0 / \ 1
+LeafBucket   InternalBucket
+                 0 / \ 1
+         LeafBucket   InternalBucket
+                          0 / \ 1
+              InternalBucket   LeafBucket
+                  0 / \ 1
+          LeafBucket   InternalBucket
+                            ...etc
+```
+
+This ensures that the closer we get to the node's KadID, the more peers the
+trie contains, so we know about more of the network.
+
+This attempts to balance knowledge of the network with maintenance overhead,
+since we will need to periodically contact peers in the routing table to
+ensure that they are still online.
+
+The `prefixLength` parameter controls how deep the trie will grow await from
+the KadID, and the `selfPrefixLength` parameter controls how deep it will
+grow towards the KadID.
+
+Larger values will result in a bigger trie which in turn causes more memory
+consumption and more network requests as the nodes are re-contacted.
+
+Setting these to the same value will create a balanced trie which will result
+in queries with fewer hops but at the cost of higher maintenance overhead.
+
 # Install
 
 ```console

packages/kad-dht/src/index.ts

@@ -78,6 +78,45 @@
  *
  * console.info(peerInfo) // peer id, multiaddrs
  * ```
+ *
+ * ## The routing table
+ *
+ * This module uses a binary trie for it's routing table. By default the trie
+ * will only grow in the direction of the KadID of the current peer:
+ *
+ * ```
+ * Peer KadID: 01101...
+ *
+ *             InternalBucket
+ *                 0 / \ 1
+ *     InternalBucket   LeafBucket
+ *         0 / \ 1
+ * LeafBucket   InternalBucket
+ *                  0 / \ 1
+ *          LeafBucket   InternalBucket
+ *                           0 / \ 1
+ *               InternalBucket   LeafBucket
+ *                   0 / \ 1
+ *           LeafBucket   InternalBucket
+ *                             ...etc
+ * ```
+ *
+ * This ensures that the closer we get to the node's KadID, the more peers the
+ * trie contains, so we know about more of the network.
+ *
+ * This attempts to balance knowledge of the network with maintenance overhead,
+ * since we will need to periodically contact peers in the routing table to
+ * ensure that they are still online.
+ *
+ * The `prefixLength` parameter controls how deep the trie will grow await from
+ * the KadID, and the `selfPrefixLength` parameter controls how deep it will
+ * grow towards the KadID.
+ *
+ * Larger values will result in a bigger trie which in turn causes more memory
+ * consumption and more network requests as the nodes are re-contacted.
+ *
+ * Setting these to the same value will create a balanced trie which will result
+ * in queries with fewer hops but at the cost of higher maintenance overhead.
  */
 
 import { KadDHT as KadDHTClass } from './kad-dht.js'

packages/kad-dht/src/routing-table/index.ts

@@ -15,7 +15,8 @@ import type { AdaptiveTimeoutInit } from '@libp2p/utils/adaptive-timeout'
 export const KAD_CLOSE_TAG_NAME = 'kad-close'
 export const KAD_CLOSE_TAG_VALUE = 50
 export const KBUCKET_SIZE = 20
-export const PREFIX_LENGTH = 7
+export const PREFIX_LENGTH = 8
+export const SELF_PREFIX_LENGTH = 32
 export const PING_NEW_CONTACT_TIMEOUT = 2000
 export const PING_NEW_CONTACT_CONCURRENCY = 20
 export const PING_NEW_CONTACT_MAX_QUEUE_SIZE = 100
@@ -33,6 +34,7 @@ export interface RoutingTableInit {
   logPrefix: string
   protocol: string
   prefixLength?: number
+  selfPrefixLength?: number
   splitThreshold?: number
   kBucketSize?: number
   pingNewContactTimeout?: AdaptiveTimeoutInit
@@ -143,6 +145,7 @@ export class RoutingTable extends TypedEventEmitter<RoutingTableEvents> implemen
     this.kb = new KBucket({
       kBucketSize: init.kBucketSize,
       prefixLength: init.prefixLength,
+      selfPrefixLength: init.selfPrefixLength,
       splitThreshold: init.splitThreshold,
       numberOfOldContactsToPing: init.numberOfOldContactsToPing,
       lastPingThreshold: init.lastPingThreshold,

packages/kad-dht/src/routing-table/k-bucket.ts

@@ -5,7 +5,7 @@ import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { xor as uint8ArrayXor } from 'uint8arrays/xor'
 import { PeerDistanceList } from '../peer-list/peer-distance-list.js'
 import { convertPeerId } from '../utils.js'
-import { KBUCKET_SIZE, LAST_PING_THRESHOLD, PING_OLD_CONTACT_COUNT, PREFIX_LENGTH } from './index.js'
+import { KBUCKET_SIZE, LAST_PING_THRESHOLD, PING_OLD_CONTACT_COUNT, PREFIX_LENGTH, SELF_PREFIX_LENGTH } from './index.js'
 import type { PeerId } from '@libp2p/interface'
 import type { AbortOptions } from 'it-protobuf-stream'
 
@@ -58,10 +58,19 @@ export interface KBucketOptions {
    * this value, the deeper the tree will grow and the slower the lookups will
    * be but the peers returned will be more specific to the key.
    *
-   * @default 32
+   * @default 8
    */
   prefixLength?: number
 
+  /**
+   * For the self key we let the trie grow up to this depth in order to store
+   * more specific peers as we get closer to our own KadID. The final leaf
+   * should contain the 20x Kad-closest peers known on the network.
+   *
+   * @default 32
+   */
+  selfPrefixLength?: number
+
   /**
    * The number of nodes that a max-depth k-bucket can contain before being
    * full.
@@ -135,6 +144,7 @@ export class KBucket {
   public root: Bucket
   public localPeer?: Peer
   private readonly prefixLength: number
+  private readonly selfPrefixLength: number
   private readonly splitThreshold: number
   private readonly kBucketSize: number
   private readonly numberOfNodesToPing: number
@@ -148,6 +158,7 @@ export class KBucket {
 
   constructor (options: KBucketOptions) {
     this.prefixLength = options.prefixLength ?? PREFIX_LENGTH
+    this.selfPrefixLength = options.selfPrefixLength ?? SELF_PREFIX_LENGTH
     this.kBucketSize = options.kBucketSize ?? KBUCKET_SIZE
     this.splitThreshold = options.splitThreshold ?? this.kBucketSize
     this.numberOfNodesToPing = options.numberOfOldContactsToPing ?? PING_OLD_CONTACT_COUNT
@@ -181,6 +192,11 @@ export class KBucket {
    * Adds a contact to the trie
    */
   async add (peerId: PeerId, options?: AbortOptions): Promise<void> {
+    // do not add self peer to trie
+    if (peerId.equals(this.localPeer?.peerId)) {
+      return
+    }
+
     const peer = {
       peerId,
       kadId: await convertPeerId(peerId),
@@ -202,6 +218,18 @@ export class KBucket {
     }
   }
 
+  private _canSplit (bucket: LeafBucket): boolean {
+    if (bucket.peers.length !== this.splitThreshold) {
+      return false
+    }
+
+    if (bucket.containsSelf === true) {
+      return bucket.depth < this.selfPrefixLength
+    }
+
+    return bucket.depth < this.prefixLength
+  }
+
   private async _add (peer: Peer, options?: AbortOptions): Promise<void> {
     const bucket = this._determineBucket(peer.kadId)
 
@@ -211,7 +239,7 @@ export class KBucket {
     }
 
     // are there too many peers in the bucket and can we make the trie deeper?
-    if (bucket.peers.length === this.splitThreshold && bucket.depth < this.prefixLength) {
+    if (this._canSplit(bucket)) {
       // split the bucket
       await this._split(bucket)
 
@@ -411,14 +439,13 @@ export class KBucket {
    */
   private _determineBucket (kadId: Uint8Array): LeafBucket {
     const bitString = uint8ArrayToString(kadId, 'base2')
-    const prefix = bitString.substring(0, this.prefixLength)
 
     function findBucket (bucket: Bucket, bitIndex: number = 0): LeafBucket {
       if (isLeafBucket(bucket)) {
         return bucket
       }
 
-      const bit = prefix[bitIndex]
+      const bit = bitString[bitIndex]
 
       if (bit === '0') {
         return findBucket(bucket.left, bitIndex + 1)
@@ -448,25 +475,23 @@ export class KBucket {
    * @param {any} bucket - bucket for splitting
    */
   private async _split (bucket: LeafBucket): Promise<void> {
-    const depth = bucket.prefix === '' ? bucket.depth : bucket.depth + 1
-
     // create child buckets
     const left: LeafBucket = {
       prefix: '0',
-      depth,
+      depth: bucket.depth + 1,
       peers: []
     }
     const right: LeafBucket = {
       prefix: '1',
-      depth,
+      depth: bucket.depth + 1,
       peers: []
     }
 
     if (bucket.containsSelf === true && this.localPeer != null) {
       delete bucket.containsSelf
       const selfNodeBitString = uint8ArrayToString(this.localPeer.kadId, 'base2')
 
-      if (selfNodeBitString[depth] === '0') {
+      if (selfNodeBitString[bucket.depth] === '0') {
         left.containsSelf = true
       } else {
         right.containsSelf = true
@@ -477,7 +502,7 @@ export class KBucket {
     for (const peer of bucket.peers) {
       const bitString = uint8ArrayToString(peer.kadId, 'base2')
 
-      if (bitString[depth] === '0') {
+      if (bitString[bucket.depth] === '0') {
         left.peers.push(peer)
         await this.onMove?.(peer, bucket, left)
       } else {
@@ -497,6 +522,11 @@ function convertToInternalBucket (bucket: any, left: any, right: any): bucket is
   bucket.left = left
   bucket.right = right
 
+  if (bucket.prefix === '') {
+    delete bucket.depth
+    delete bucket.prefix
+  }
+
   return true
 }