Return undefined for candles orderbookMidPrice instead of hitting OrderbookCache in Redis

.github/workflows/indexer-build-docker-image-check.yml

@@ -77,3 +77,33 @@ jobs:
           DOCKER_BUILDKIT=1 docker build \
             --platform amd64 \
             -f Dockerfile.bazooka.remote .
+
+  check-build-auxo:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./indexer
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+
+      - name: Install pnpm
+        run:
+          npm install -g [email protected]
+
+      - name: Build and install
+        run: |
+          pnpm install --loglevel warn --frozen-lockfile
+          pnpm run build:prod:all
+
+      - name: Build docker image for auxo
+        id: build-image
+        run: |
+          DOCKER_BUILDKIT=1 docker build \
+            --platform amd64 \
+            -f Dockerfile.auxo.remote .

indexer/Dockerfile.auxo.remote

@@ -11,7 +11,11 @@ COPY ./patches ./patches
 
 # Copy auxo and imported packages
 COPY ./packages/base/ ./packages/base/
+COPY ./packages/dev/ ./packages/dev/
+COPY ./packages/postgres/ ./packages/postgres/
 COPY ./packages/v4-protos/ ./packages/v4-protos/
+COPY ./packages/kafka/ ./packages/kafka/
+COPY ./packages/redis/ ./packages/redis/
 COPY ./services/auxo/ ./services/auxo/
 COPY ./packages/v4-proto-parser/ ./packages/v4-proto-parser/
 

indexer/Dockerfile.bazooka.remote

@@ -11,6 +11,7 @@ COPY ./patches ./patches
 
 # Copy bazooka and imported packages
 COPY ./packages/base/ ./packages/base/
+COPY ./packages/dev/ ./packages/dev/
 COPY ./packages/postgres/ ./packages/postgres/
 COPY ./packages/v4-protos/ ./packages/v4-protos/
 COPY ./packages/kafka/ ./packages/kafka/

indexer/packages/base/__tests__/instance-id.test.ts

@@ -0,0 +1,87 @@
+import { setInstanceId, getInstanceId, resetForTests } from '../src/instance-id';
+import { axiosRequest } from '../src/axios';
+import { asMock } from '@dydxprotocol-indexer/dev';
+import logger from '../src/logger';
+import config from '../src/config';
+
+jest.mock('../src/axios', () => ({
+  ...(jest.requireActual('../src/axios') as object),
+  axiosRequest: jest.fn(),
+}));
+
+describe('instance-id', () => {
+  describe('setInstanceId', () => {
+    const defaultTaskArn = 'defaultTaskArn';
+    const defaultResponse = {
+      TaskARN: defaultTaskArn,
+    };
+    const ecsUrl = config.ECS_CONTAINER_METADATA_URI_V4;
+
+    beforeEach(() => {
+      config.ECS_CONTAINER_METADATA_URI_V4 = ecsUrl;
+      resetForTests();
+      jest.resetAllMocks();
+      jest.restoreAllMocks();
+      asMock(axiosRequest).mockResolvedValue(defaultResponse);
+    });
+
+    afterAll(() => {
+      jest.clearAllMocks();
+      jest.restoreAllMocks();
+    });
+
+    it('should set instance id to task ARN in staging', async () => {
+      jest.spyOn(config, 'isStaging').mockReturnValueOnce(true);
+      config.ECS_CONTAINER_METADATA_URI_V4 = 'url';
+      await setInstanceId();
+
+      expect(getInstanceId()).toEqual(defaultTaskArn);
+    });
+
+    it('should set instance id to task ARN in production', async () => {
+      jest.spyOn(config, 'isProduction').mockReturnValueOnce(true);
+      config.ECS_CONTAINER_METADATA_URI_V4 = 'url';
+      await setInstanceId();
+
+      expect(getInstanceId()).toEqual(defaultTaskArn);
+    });
+
+    it('should not call metadata endpoint if not production or staging', async () => {
+      config.ECS_CONTAINER_METADATA_URI_V4 = 'url';
+      await setInstanceId();
+
+      expect(getInstanceId()).not.toEqual(defaultTaskArn);
+      expect(asMock(axiosRequest)).not.toHaveBeenCalled();
+    });
+
+    it('should not set instance id if already set', async () => {
+      jest.spyOn(config, 'isStaging').mockReturnValue(true);
+      config.ECS_CONTAINER_METADATA_URI_V4 = 'url';
+      await setInstanceId();
+      const instanceId = getInstanceId();
+      await setInstanceId();
+
+      expect(getInstanceId()).toEqual(instanceId);
+      expect(axiosRequest).toHaveBeenCalledTimes(1);
+    });
+
+    it('should log error and set instance id to uuid if request errors', async () => {
+      jest.spyOn(config, 'isStaging').mockReturnValue(true);
+      config.ECS_CONTAINER_METADATA_URI_V4 = 'url';
+      const loggerErrorSpy = jest.spyOn(logger, 'error');
+      const emptyInstanceId = getInstanceId();
+      asMock(axiosRequest).mockRejectedValueOnce(new Error());
+      await setInstanceId();
+
+      expect(loggerErrorSpy).toHaveBeenCalledTimes(1);
+      expect(getInstanceId()).not.toEqual(emptyInstanceId);
+    });
+
+    it('should not call metadata endpoint if url is empty', async () => {
+      jest.spyOn(config, 'isStaging').mockReturnValue(true);
+      await setInstanceId();
+
+      expect(axiosRequest).not.toHaveBeenCalled();
+    });
+  });
+});

indexer/packages/base/package.json

@@ -2,14 +2,15 @@
   "name": "@dydxprotocol-indexer/base",
   "version": "0.0.1",
   "description": "",
-  "main": "build/index.js",
+  "main": "build/src/index.js",
   "devDependencies": {
     "@dydxprotocol-indexer/dev": "workspace:^0.0.1",
     "@types/big.js": "^6.1.5",
     "@types/express": "^4.17.13",
     "@types/lodash": "^4.14.182",
     "@types/traverse": "^0.6.32",
     "express": "^4.18.1",
+    "jest": "^28.1.2",
     "typescript": "^4.7.4"
   },
   "scripts": {
@@ -18,7 +19,7 @@
     "build": "rm -rf build/ && tsc",
     "build:prod": "pnpm run build",
     "build:watch": "pnpm run build -- --watch",
-    "test": "echo \"Error: no test specified\""
+    "test": "NODE_ENV=test jest --runInBand --forceExit"
   },
   "repository": {
     "type": "git",
@@ -31,13 +32,15 @@
   },
   "homepage": "https://github.com/dydxprotocol/indexer#readme",
   "dependencies": {
+    "@aws-sdk/client-ec2": "^3.354.0",
     "axios": "^1.2.1",
     "big.js": "^6.2.1",
     "bignumber.js": "^9.0.2",
     "dotenv-flow": "^3.2.0",
     "hot-shots": "^9.1.0",
     "lodash": "^4.17.21",
     "traverse": "^0.6.6",
+    "uuid": "^8.3.2",
     "winston": "^3.8.1",
     "winston-transport": "^4.5.0",
     "@bugsnag/core": "^7.18.0",

indexer/packages/base/src/axios/errors.ts

@@ -1,8 +1,8 @@
 import { WrappedError } from '../errors';
 
 export interface AxiosOriginalError extends Error {
-  isAxiosError: true;
-  toJSON(): Error;
+  isAxiosError: true,
+  toJSON(): Error,
 }
 
 export interface AxiosErrorResponse {

indexer/packages/base/src/az-id.ts

@@ -0,0 +1,51 @@
+import { DescribeAvailabilityZonesCommand, EC2Client } from '@aws-sdk/client-ec2';
+
+import { axiosRequest } from './axios';
+import config from './config';
+import logger from './logger';
+
+export async function getAvailabilityZoneId(): Promise<string> {
+  if (config.ECS_CONTAINER_METADATA_URI_V4 !== '' && config.AWS_REGION !== '') {
+    const taskUrl = `${config.ECS_CONTAINER_METADATA_URI_V4}/task`;
+    try {
+      const response = await axiosRequest({
+        method: 'GET',
+        url: taskUrl,
+      }) as { AvailabilityZone: string };
+      const client = new EC2Client({ region: config.AWS_REGION });
+      const command = new DescribeAvailabilityZonesCommand({
+        ZoneNames: [response.AvailabilityZone],
+      });
+      try {
+        const ec2Response = await client.send(command);
+        const zoneId = ec2Response.AvailabilityZones![0].ZoneId!;
+        logger.info({
+          at: 'az-id#getAvailabilityZoneId',
+          message: `Got availability zone id ${zoneId}.`,
+        });
+        return ec2Response.AvailabilityZones![0].ZoneId!;
+      } catch (error) {
+        logger.error({
+          at: 'az-id#getAvailabilityZoneId',
+          message: 'Failed to fetch availabilty zone id from EC2. ',
+          error,
+        });
+        return '';
+      }
+    } catch (error) {
+      logger.error({
+        at: 'az-id#getAvailabilityZoneId',
+        message: 'Failed to retrieve availability zone from metadata endpoint. No availabilty zone id found.',
+        error,
+        taskUrl,
+      });
+      return '';
+    }
+  } else {
+    logger.error({
+      at: 'az-id#getAvailabilityZoneId',
+      message: 'No metadata URI or region. No availabilty zone id found.',
+    });
+    return '';
+  }
+}

indexer/packages/base/src/config-util.ts

@@ -48,10 +48,10 @@ interface ParseOptions<T> {
   // If `default` is present, then the environment variable will be optional and will default to the
   // value of `default` when unset. In particular, `default` may be null in which case the config
   // value will be null when the environment variable is not set.
-  default: T;
+  default: T,
 
   // Can be specified to ensure the default value is not used when running in a certain NODE_ENV.
-  requireInEnv?: NodeEnv[];
+  requireInEnv?: NodeEnv[],
 }
 
 const NODE_ENV = process.env.NODE_ENV;
@@ -229,10 +229,10 @@ export function parseSchema<T extends SchemaBase>(
 ): {
   [K in keyof T]: T[K] extends ParseFn<infer U> ? U : never;
 } & {
-  isDevelopment: () => boolean;
-  isStaging: () => boolean;
-  isProduction: () => boolean;
-  isTest: () => boolean;
+  isDevelopment: () => boolean,
+  isStaging: () => boolean,
+  isProduction: () => boolean,
+  isTest: () => boolean,
 } {
   const config = _.mapValues(schema, (parseFn: ParseFn<T>, varName: string) => {
     const fullVarName = prefix ? `${prefix}_${varName}` : varName;

indexer/packages/base/src/config.ts

@@ -30,14 +30,16 @@ export const baseConfigSchema = {
   SEND_BUGSNAG_ERRORS: parseBoolean({
     default: true,
   }),
-  SERVICE_NAME: parseString(),
+  SERVICE_NAME: parseString({ default: '' }),
 
   // Optional environment variables.
   NODE_ENV: parseString({ default: null }),
   ENABLE_LOGS_IN_TEST: parseBoolean({ default: false }),
   STATSD_HOST: parseString({ default: 'localhost' }),
   STATSD_PORT: parseInteger({ default: 8125 }),
   LOG_LEVEL: parseString({ default: 'debug' }),
+  ECS_CONTAINER_METADATA_URI_V4: parseString({ default: '' }),
+  AWS_REGION: parseString({ default: '' }),
 };
 
 export default parseSchema(baseConfigSchema);

indexer/packages/base/src/index.ts

@@ -13,6 +13,8 @@ export * from './constants';
 export * from './bugsnag';
 export * from './stats-util';
 export * from './date-helpers';
+export * from './instance-id';
+export * from './az-id';
 
 // Do this outside logger.ts to avoid a dependency cycle with logger transports that may trigger
 // additional logging.

indexer/packages/base/src/instance-id.ts

@@ -0,0 +1,51 @@
+import { v4 as uuidv4 } from 'uuid';
+
+import { axiosRequest } from './axios';
+import config from './config';
+import logger from './logger';
+
+let INSTANCE_ID: string = '';
+
+export function getInstanceId(): string {
+  return INSTANCE_ID;
+}
+
+export async function setInstanceId(): Promise<void> {
+  if (INSTANCE_ID !== '') {
+    return;
+  }
+  if (config.ECS_CONTAINER_METADATA_URI_V4 !== '' &&
+      (
+        config.isProduction() || config.isStaging()
+      )
+  ) {
+    // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v4.html
+    const taskUrl = `${config.ECS_CONTAINER_METADATA_URI_V4}/task`;
+    try {
+      const response = await axiosRequest({
+        method: 'GET',
+        url: taskUrl,
+      }) as { TaskARN: string };
+      INSTANCE_ID = response.TaskARN;
+    } catch (error) {
+      logger.error({
+        at: 'instance-id#setInstanceId',
+        message: 'Failed to retrieve task arn from metadata endpoint. Falling back to uuid.',
+        error,
+        taskUrl,
+      });
+      INSTANCE_ID = uuidv4();
+    }
+  } else {
+    INSTANCE_ID = uuidv4();
+
+  }
+}
+
+// Exported for tests
+export function resetForTests(): void {
+  if (!config.isTest()) {
+    throw new Error(`resetForTests() cannot be called for env: ${config.NODE_ENV}`);
+  }
+  INSTANCE_ID = '';
+}

indexer/packages/base/src/logger.ts

@@ -13,21 +13,21 @@ type UnusedLevels = 'warn' | 'help' | 'data' | 'prompt' | 'http' | 'verbose' | '
 
 // Enforce type constraints on the objects passed into Winston logging functions.
 interface LeveledLogMethod {
-  (infoObject: InfoObject): winston.Logger;
+  (infoObject: InfoObject): winston.Logger,
 }
 // Exclude the functions whose type we want to change from the base definition. This seems to be
 // enough (and the only way I've found) to trick TypeScript into accepting the modified LoggerExport
 // as a valid extension of the base winston.Logger type.
 type SyslogLevels = 'emerg' | 'alert' | 'crit' | 'error' | 'warning' | 'notice' | 'info' | 'debug';
 export interface LoggerExport extends Omit<winston.Logger, UnusedLevels | SyslogLevels> {
-  emerg: LeveledLogMethod;
-  alert: LeveledLogMethod;
-  crit: LeveledLogMethod;
-  error: LeveledLogMethod;
-  warning: LeveledLogMethod;
-  notice: LeveledLogMethod;
-  info: LeveledLogMethod;
-  debug: LeveledLogMethod;
+  emerg: LeveledLogMethod,
+  alert: LeveledLogMethod,
+  crit: LeveledLogMethod,
+  error: LeveledLogMethod,
+  warning: LeveledLogMethod,
+  notice: LeveledLogMethod,
+  info: LeveledLogMethod,
+  debug: LeveledLogMethod,
 }
 
 const logger: LoggerExport = winston.createLogger({

indexer/packages/base/src/types.ts

@@ -21,8 +21,8 @@ export enum BugsnagReleaseStage {
 }
 
 export interface PagerDutyInfo {
-  message: {};
-  id?: string;
+  message: {},
+  id?: string,
 }
 
 // Enforce type constraints on the objects passed into Winston logging functions.