Merge pull request #289 from MatrixAI/seed-nodes

Supporting trusted seed nodes (`nodes` domain refactoring, static sources for seed nodes)

.gitignore

@@ -123,3 +123,4 @@ dist
 
 # editor
 .vscode/
+.idea/

src/PolykeyAgent.ts

@@ -1,6 +1,7 @@
 import type { FileSystem } from './types';
 import type { PolykeyWorkerManagerInterface } from './workers/types';
 import type { Host, Port } from './network/types';
+import type { NodeMapping } from './nodes/types';
 
 import path from 'path';
 import process from 'process';
@@ -58,6 +59,7 @@ class PolykeyAgent {
     networkConfig = {},
     forwardProxyConfig = {},
     reverseProxyConfig = {},
+    seedNodes = {},
     // Optional dependencies
     status,
     schema,
@@ -99,6 +101,7 @@ class PolykeyAgent {
       connTimeoutTime?: number;
     };
     networkConfig?: NetworkConfig;
+    seedNodes?: NodeMapping;
     status?: Status;
     schema?: Schema;
     keyManager?: KeyManager;
@@ -241,6 +244,7 @@ class PolykeyAgent {
       nodeManager ??
       (await NodeManager.createNodeManager({
         db,
+        seedNodes,
         sigchain,
         keyManager,
         fwdProxy,
@@ -503,6 +507,8 @@ class PolykeyAgent {
     });
 
     await this.nodeManager.start({ fresh });
+    await this.nodeManager.getConnectionsToSeedNodes();
+    await this.nodeManager.syncNodeGraph();
     await this.vaultManager.start({ fresh });
     await this.notificationsManager.start({ fresh });
     await this.sessionManager.start({ fresh });

src/agent/agentService.ts

@@ -195,7 +195,7 @@ function createAgentService({
         );
         for (const node of closestNodes) {
           const addressMessage = new nodesPB.Address();
-          addressMessage.setHost(node.address.ip);
+          addressMessage.setHost(node.address.host);
           addressMessage.setPort(node.address.port);
           // Add the node to the response's map (mapping of node ID -> node address)
           response.getNodeTableMap().set(node.id, addressMessage);

src/bin/agent/CommandStart.ts

@@ -23,6 +23,9 @@ class CommandStart extends CommandPolykey {
     this.addOption(binOptions.clientPort);
     this.addOption(binOptions.ingressHost);
     this.addOption(binOptions.ingressPort);
+    this.addOption(binOptions.connTimeoutTime);
+    this.addOption(binOptions.seedNodes);
+    this.addOption(binOptions.network);
     this.addOption(binOptions.background);
     this.addOption(binOptions.backgroundOutFile);
     this.addOption(binOptions.backgroundErrFile);
@@ -66,19 +69,28 @@ class CommandStart extends CommandPolykey {
         options.recoveryCodeFile,
         this.fs,
       );
+      const [seedNodes, defaults] = options.seedNodes;
+      if (defaults) Object.assign(seedNodes, options.network);
       const agentConfig = {
         password,
         nodePath: options.nodePath,
         keysConfig: {
           rootKeyPairBits: options.rootKeyPairBits,
           recoveryCode: recoveryCodeIn,
         },
+        forwardProxyConfig: {
+          connTimeoutTime: options.connTimeoutTime,
+        },
+        reverseProxyConfig: {
+          connTimeoutTime: options.connTimeoutTime,
+        },
         networkConfig: {
           clientHost: options.clientHost,
           clientPort: options.clientPort,
           ingressHost: options.ingressHost,
           ingressPort: options.ingressPort,
         },
+        seedNodes,
         fresh: options.fresh,
       };
       let recoveryCodeOut: RecoveryCode | undefined;

src/bin/utils/options.ts

@@ -4,7 +4,7 @@
  * @module
  */
 import commander from 'commander';
-import * as binParsers from './parsers';
+import * as binParsers from '../utils/parsers';
 import config from '../../config';
 
 /**
@@ -83,6 +83,13 @@ const ingressPort = new commander.Option(
   .env('PK_INGRESS_PORT')
   .default(config.defaults.networkConfig.ingressPort);
 
+const connTimeoutTime = new commander.Option(
+  '--connection-timeout <ms>',
+  'Timeout value for connection establishment between nodes',
+)
+  .argParser(binParsers.parseNumber)
+  .default(config.defaults.forwardProxyConfig.connTimeoutTime);
+
 const passwordFile = new commander.Option(
   '-pf, --password-file <path>',
   'Path to Password',
@@ -118,6 +125,22 @@ const rootKeyPairBits = new commander.Option(
   'Bit size of root key pair',
 ).argParser(binParsers.parseNumber);
 
+const seedNodes = new commander.Option(
+  '-sn, --seed-nodes [nodeId1@host:port;nodeId2@host:port;...]',
+  'Seed node address mappings',
+)
+  .argParser(binParsers.parseSeedNodes)
+  .env('PK_SEED_NODES')
+  .default([{}, true]);
+
+const network = new commander.Option(
+  '-n --network <network>',
+  'Setting the desired default network.',
+)
+  .argParser(binParsers.parseNetwork)
+  .env('PK_NETWORK')
+  .default(config.defaults.network.mainnet);
+
 export {
   nodePath,
   format,
@@ -128,11 +151,14 @@ export {
   clientPort,
   ingressHost,
   ingressPort,
+  connTimeoutTime,
+  recoveryCodeFile,
   passwordFile,
   passwordNewFile,
-  recoveryCodeFile,
   background,
   backgroundOutFile,
   backgroundErrFile,
   rootKeyPairBits,
+  seedNodes,
+  network,
 };

src/bin/utils/parsers.ts

@@ -1,6 +1,11 @@
 import type { IdentityId, ProviderId } from '../../identities/types';
+import type { Host, Hostname, Port } from '../../network/types';
+import type { NodeAddress, NodeId, NodeMapping } from '../../nodes/types';
 import commander from 'commander';
 import * as nodesUtils from '../../nodes/utils';
+import * as networkUtils from '../../network/utils';
+import config from '../../config';
+import { never } from '../../utils';
 
 function parseNumber(v: string): number {
   const num = parseInt(v);
@@ -56,4 +61,96 @@ function parseIdentityString(identityString: string): {
   return { providerId, identityId };
 }
 
-export { parseNumber, parseSecretPath, parseGestaltId };
+/**
+ * Acquires the default seed nodes from src/config.ts.
+ */
+function getDefaultSeedNodes(network: string): NodeMapping {
+  const seedNodes: NodeMapping = {};
+  let source;
+  switch (network) {
+    case 'testnet':
+      source = config.defaults.network.testnet;
+      break;
+    case 'mainnet':
+      source = config.defaults.network.mainnet;
+      break;
+    default:
+      never();
+  }
+  for (const id in source) {
+    const seedNodeId = id as NodeId;
+    const seedNodeAddress: NodeAddress = {
+      host: source[seedNodeId].host as Host | Hostname,
+      port: source[seedNodeId].port as Port,
+    };
+    seedNodes[seedNodeId] = seedNodeAddress;
+  }
+  return seedNodes;
+}
+
+/**
+ * Seed nodes expected to be of form 'nodeId1@host:port;nodeId2@host:port;...'
+ * By default, any specified seed nodes (in CLI option, or environment variable)
+ * will overwrite the default nodes in src/config.ts.
+ * Special flag '<seed-nodes>' in the content indicates that the default seed
+ * nodes should be added to the starting seed nodes instead of being overwritten.
+ */
+function parseSeedNodes(rawSeedNodes: string): [NodeMapping, boolean] {
+  const seedNodeMappings: NodeMapping = {};
+  let defaults = false;
+  // If specifically set no seed nodes, then ensure we start with none
+  if (rawSeedNodes === '') return [seedNodeMappings, defaults];
+  const semicolonSeedNodes = rawSeedNodes.split(';');
+  for (const rawSeedNode of semicolonSeedNodes) {
+    // Empty string will occur if there's an extraneous ';' (e.g. at end of env)
+    if (rawSeedNode === '') continue;
+    // Append the default seed nodes if we encounter the special flag
+    if (rawSeedNode === '<default>') {
+      defaults = true;
+      continue;
+    }
+    const idHostPort = rawSeedNode.split(/[@:]/);
+    if (idHostPort.length !== 3) {
+      throw new commander.InvalidOptionArgumentError(
+        `${rawSeedNode} is not of format 'nodeId@host:port'`,
+      );
+    }
+    if (!nodesUtils.isNodeId(idHostPort[0])) {
+      throw new commander.InvalidOptionArgumentError(
+        `${idHostPort[0]} is not a valid node ID`,
+      );
+    }
+    if (!networkUtils.isValidHostname(idHostPort[1])) {
+      throw new commander.InvalidOptionArgumentError(
+        `${idHostPort[1]} is not a valid hostname`,
+      );
+    }
+    const port = parseNumber(idHostPort[2]);
+    const seedNodeId = idHostPort[0] as NodeId;
+    const seedNodeAddress: NodeAddress = {
+      host: idHostPort[1] as Host | Hostname,
+      port: port as Port,
+    };
+    seedNodeMappings[seedNodeId] = seedNodeAddress;
+  }
+  return [seedNodeMappings, defaults];
+}
+
+function parseNetwork(network: string): NodeMapping {
+  // Getting a list of network names from the config defaults
+  const networks = config.defaults.network;
+  const validNetworks = Object.keys(networks);
+
+  // Checking if the network name is valid.
+  if (validNetworks.includes(network)) return getDefaultSeedNodes(network);
+  throw new commander.InvalidArgumentError(`${network} is not a valid network`);
+}
+
+export {
+  parseNumber,
+  parseSecretPath,
+  parseGestaltId,
+  getDefaultSeedNodes,
+  parseSeedNodes,
+  parseNetwork,
+};

src/client/clientService.ts

@@ -14,7 +14,6 @@ import type { FileSystem } from '../types';
 
 import type * as grpc from '@grpc/grpc-js';
 import type { IClientServiceServer } from '../proto/js/polykey/v1/client_service_grpc_pb';
-import { promisify } from 'util';
 import createStatusRPC from './rpcStatus';
 import createSessionsRPC from './rpcSessions';
 import createVaultRPC from './rpcVaults';
@@ -25,7 +24,6 @@ import createIdentitiesRPC from './rpcIdentities';
 import createNotificationsRPC from './rpcNotifications';
 import * as clientUtils from './utils';
 import * as grpcUtils from '../grpc/utils';
-import * as nodesPB from '../proto/js/polykey/v1/nodes/nodes_pb';
 import * as utilsPB from '../proto/js/polykey/v1/utils/utils_pb';
 import { ClientServiceService } from '../proto/js/polykey/v1/client_service_grpc_pb';
 
@@ -114,17 +112,6 @@ function createClientService({
       notificationsManager,
       authenticate,
     }),
-    nodesList: async (
-      call: grpc.ServerWritableStream<utilsPB.EmptyMessage, nodesPB.Node>,
-    ): Promise<void> => {
-      // Call.request // PROCESS THE REQEUST MESSAGE
-      const nodeMessage = new nodesPB.Node();
-      nodeMessage.setNodeId('some node name');
-      const write = promisify(call.write).bind(call);
-      await write(nodeMessage);
-      call.end();
-      return;
-    },
     agentStop: async (
       call: grpc.ServerUnaryCall<utilsPB.EmptyMessage, utilsPB.EmptyMessage>,
       callback: grpc.sendUnaryData<utilsPB.EmptyMessage>,

src/client/rpcNodes.ts

@@ -7,10 +7,9 @@ import type * as grpc from '@grpc/grpc-js';
 import type * as utils from '../client/utils';
 import * as utilsPB from '../proto/js/polykey/v1/utils/utils_pb';
 import * as nodesPB from '../proto/js/polykey/v1/nodes/nodes_pb';
-import * as nodesUtils from '../nodes/utils';
+import { utils as nodesUtils, errors as nodesErrors } from '../nodes';
 import * as grpcUtils from '../grpc/utils';
-import * as nodesErrors from '../nodes/errors';
-import { makeNodeId } from '../nodes/utils';
+import * as networkUtils from '../network/utils';
 
 const createNodesRPC = ({
   nodeManager,
@@ -40,16 +39,19 @@ const createNodesRPC = ({
         if (!validNodeId) {
           throw new nodesErrors.ErrorInvalidNodeId();
         }
-        const validHost = nodesUtils.isValidHost(
+        const validHost = networkUtils.isValidHost(
           call.request.getAddress()!.getHost(),
         );
         if (!validHost) {
           throw new nodesErrors.ErrorInvalidHost();
         }
-        await nodeManager.setNode(makeNodeId(call.request.getNodeId()), {
-          ip: call.request.getAddress()!.getHost(),
-          port: call.request.getAddress()!.getPort(),
-        } as NodeAddress);
+        await nodeManager.setNode(
+          nodesUtils.makeNodeId(call.request.getNodeId()),
+          {
+            host: call.request.getAddress()!.getHost(),
+            port: call.request.getAddress()!.getPort(),
+          } as NodeAddress,
+        );
         callback(null, response);
         return;
       } catch (err) {
@@ -70,7 +72,7 @@ const createNodesRPC = ({
         call.sendMetadata(metadata);
 
         const status = await nodeManager.pingNode(
-          makeNodeId(call.request.getNodeId()),
+          nodesUtils.makeNodeId(call.request.getNodeId()),
         );
         response.setSuccess(status);
         callback(null, response);
@@ -94,7 +96,7 @@ const createNodesRPC = ({
         const metadata = await authenticate(call.metadata);
         call.sendMetadata(metadata);
 
-        const remoteNodeId = makeNodeId(call.request.getNodeId());
+        const remoteNodeId = nodesUtils.makeNodeId(call.request.getNodeId());
         const gestaltInvite = await notificationsManager.findGestaltInvite(
           remoteNodeId,
         );
@@ -133,12 +135,12 @@ const createNodesRPC = ({
         const metadata = await authenticate(call.metadata);
         call.sendMetadata(metadata);
 
-        const nodeId = makeNodeId(call.request.getNodeId());
+        const nodeId = nodesUtils.makeNodeId(call.request.getNodeId());
         const address = await nodeManager.findNode(nodeId);
         response
           .setNodeId(nodeId)
           .setAddress(
-            new nodesPB.Address().setHost(address.ip).setPort(address.port),
+            new nodesPB.Address().setHost(address.host).setPort(address.port),
           );
         callback(null, response);
         return;

src/config.ts

@@ -91,6 +91,21 @@ const config = {
       connConnectTime: 20000,
       connTimeoutTime: 20000,
     },
+    // Note: this is not used by the `PolykeyAgent`, that is defaulting to `{}`.
+    network: {
+      mainnet: {
+        v359vgrgmqf1r5g4fvisiddjknjko6bmm4qv7646jr7fi9enbfuug: {
+          host: 'testnet.polykey.io',
+          port: 1314,
+        },
+      },
+      testnet: {
+        v359vgrgmqf1r5g4fvisiddjknjko6bmm4qv7646jr7fi9enbfuug: {
+          host: '127.0.0.3',
+          port: 1314,
+        },
+      },
+    },
   },
 };