Merge branch 'dev' into animal-sex

Makefile

@@ -26,8 +26,7 @@ backend: | close build-backend run-backend ## Performs all commands necessary to
 web: | close build-web check-env run-web ## Performs all commands necessary to run all backend+web projects (db, api, app) in docker
 
 db-setup: | build-db-setup run-db-setup ## Performs all commands necessary to run the database migrations and seeding
-db-migrate: | build-db-migrate run-db-migrate ## Performs all commands necessary to run the database migrations
-db-rollback: | build-db-rollback run-db-rollback ## Performs all commands necessary to rollback the latest database migrations
+
 clamav: | build-clamav run-clamav ## Performs all commands necessary to run clamav
 
 fix: | lint-fix format-fix ## Performs both lint-fix and format-fix commands
@@ -158,30 +157,6 @@ run-db-setup: ## Run the database migrations and seeding
 	@echo "==============================================="
 	@docker compose up db_setup
 
-build-db-migrate: ## Build the db knex migrations image
-	@echo "==============================================="
-	@echo "Make: build-db-migrate - building db knex migrate image"
-	@echo "==============================================="
-	@docker compose build db_migrate
-
-run-db-migrate: ## Run the database migrations
-	@echo "==============================================="
-	@echo "Make: run-db-migrate - running database migrations"
-	@echo "==============================================="
-	@docker compose up db_migrate
-
-build-db-rollback: ## Build the db knex rollback image
-	@echo "==============================================="
-	@echo "Make: build-db-rollback - building db knex rollback image"
-	@echo "==============================================="
-	@docker compose build db_rollback
-
-run-db-rollback: ## Rollback the latest database migrations
-	@echo "==============================================="
-	@echo "Make: run-db-rollback - rolling back the latest database migrations"
-	@echo "==============================================="
-	@docker compose up db_rollback
-
 ## ------------------------------------------------------------------------------
 ## clamav commands
 ## ------------------------------------------------------------------------------

api/package.json

@@ -27,13 +27,15 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.583.0",
+    "@aws-sdk/lib-storage": "^3.621.0",
     "@aws-sdk/s3-request-presigner": "^3.583.0",
     "@turf/bbox": "^6.5.0",
     "@turf/circle": "^6.5.0",
     "@turf/helpers": "^6.5.0",
     "@turf/meta": "^6.5.0",
     "adm-zip": "0.5.12",
     "ajv": "^8.12.0",
+    "archiver": "^7.0.1",
     "axios": "^1.6.7",
     "clamscan": "^2.2.1",
     "dayjs": "^1.11.10",
@@ -55,6 +57,7 @@
     "mime": "^3.0.0",
     "multer": "^1.4.5-lts.1",
     "pg": "^8.7.1",
+    "pg-query-stream": "^4.5.5",
     "qs": "^6.10.1",
     "sql-template-strings": "^2.2.2",
     "swagger-ui-express": "^4.3.0",
@@ -69,6 +72,7 @@
   "devDependencies": {
     "@istanbuljs/nyc-config-typescript": "^1.0.1",
     "@types/adm-zip": "^0.4.34",
+    "@types/archiver": "^6.0.2",
     "@types/chai": "^4.3.14",
     "@types/clamscan": "^2.0.8",
     "@types/express": "^4.17.13",

api/src/__mocks__/db.ts

@@ -1,5 +1,5 @@
 import { Request, Response } from 'express';
-import { QueryResult } from 'pg';
+import { PoolClient, QueryResult } from 'pg';
 import sinon from 'sinon';
 import * as db from '../database/db';
 import { IDBConnection } from '../database/db';
@@ -35,6 +35,9 @@ export const getMockDBConnection = (config?: Partial<IDBConnection>): IDBConnect
     systemUserIdentifier: () => {
       return null as unknown as string;
     },
+    getClient: async () => {
+      return null as unknown as PoolClient;
+    },
     open: async () => {
       // do nothing
     },
@@ -47,9 +50,6 @@ export const getMockDBConnection = (config?: Partial<IDBConnection>): IDBConnect
     rollback: async () => {
       // do nothing
     },
-    query: async () => {
-      return undefined as unknown as QueryResult<any>;
-    },
     sql: async () => {
       return undefined as unknown as QueryResult<any>;
     },

api/src/app.ts

@@ -5,14 +5,15 @@ import { OpenAPIV3 } from 'openapi-types';
 import path from 'path';
 import swaggerUIExperss from 'swagger-ui-express';
 import { defaultPoolConfig, initDBPool } from './database/db';
-import { ensureHTTPError, HTTP500 } from './errors/http-error';
+import { ensureHTTPError, HTTP400, HTTP500 } from './errors/http-error';
 import {
   authorizeAndAuthenticateMiddleware,
   getCritterbaseProxyMiddleware,
   replaceAuthorizationHeaderMiddleware
 } from './middleware/critterbase-proxy';
 import { rootAPIDoc } from './openapi/root-api-doc';
 import { authenticateRequest, authenticateRequestOptional } from './request-handlers/security/authentication';
+import { scanFileForVirus } from './utils/file-utils';
 import { getLogger } from './utils/logger';
 
 const defaultLog = getLogger('app');
@@ -75,7 +76,7 @@ const openAPIFramework = initialize({
     'application/json': express.json({ limit: MAX_REQ_BODY_SIZE }),
     'multipart/form-data': function (req, res, next) {
       const multerRequestHandler = multer({
-        storage: multer.memoryStorage(),
+        storage: multer.memoryStorage(), // TODO change to local/PVC storage and stream file uploads to S3?
         limits: { fileSize: MAX_UPLOAD_FILE_SIZE }
       }).array('media', MAX_UPLOAD_NUM_FILES);
 
@@ -89,11 +90,27 @@ const openAPIFramework = initialize({
        *
        * @see https://www.npmjs.com/package/express-openapi#argsconsumesmiddleware
        */
-      multerRequestHandler(req, res, (error?: any) => {
+      multerRequestHandler(req, res, async (error?: any) => {
         if (error) {
           return next(error);
         }
 
+        // Scan files for malicious content, if enabled
+        const virusScanPromises = (req.files as Express.Multer.File[]).map(async function (file) {
+          const isSafe = await scanFileForVirus(file);
+
+          if (!isSafe) {
+            throw new HTTP400('Malicious file content detected.', [{ file_name: file.originalname }]);
+          }
+        });
+
+        try {
+          await Promise.all(virusScanPromises);
+        } catch (error) {
+          // If a virus is detected, return error and do not continue
+          return next(error);
+        }
+
         // Ensure `req.files` or `req.body.media` is always set to an array
         const multerFiles = req.files ?? [];
 

api/src/database/db.test.ts

@@ -266,52 +266,6 @@ describe('db', () => {
         });
       });
 
-      describe('query', () => {
-        describe('when a connection is open', () => {
-          it('sends a query with values', async () => {
-            sinonSandbox.stub(db, 'getDBPool').returns(mockPool as unknown as pg.Pool);
-
-            await connection.open();
-
-            await connection.query('sql query', ['value1', 'value2']);
-
-            expect(queryStub).to.have.been.calledWith('sql query', ['value1', 'value2']);
-          });
-
-          it('sends a query with empty values', async () => {
-            sinonSandbox.stub(db, 'getDBPool').returns(mockPool as unknown as pg.Pool);
-
-            await connection.open();
-
-            await connection.query('sql query');
-
-            expect(queryStub).to.have.been.calledWith('sql query', []);
-          });
-        });
-
-        describe('when a connection is not open', () => {
-          it('throws an error', async () => {
-            sinonSandbox.stub(db, 'getDBPool').returns(mockPool as unknown as pg.Pool);
-
-            let expectedError: ApiExecuteSQLError;
-            try {
-              await connection.query('sql query');
-
-              expect.fail('Expected an error to be thrown');
-            } catch (error) {
-              expectedError = error as ApiExecuteSQLError;
-            }
-
-            expect(expectedError.message).to.equal('Failed to execute SQL');
-
-            expect(expectedError.errors?.length).to.be.greaterThan(0);
-            expectedError.errors?.forEach((item) => {
-              expect(item).to.be.eql({ name: 'Error', message: 'DBConnection is not open' });
-            });
-          });
-        });
-      });
-
       describe('sql', () => {
         describe('when a connection is open', () => {
           it('sends a sql statement', async () => {

api/src/database/db.ts

@@ -102,6 +102,15 @@ export const getDBPool = function (): pg.Pool | undefined {
 };
 
 export interface IDBConnection {
+  /**
+   * Get a new pg client.
+   *
+   * Note: This is not the same client that is initialized when calling `.open()`, and must be released manually by
+   * calling `client.release()`.
+   *
+   * @memberof IDBConnection
+   */
+  getClient: () => Promise<pg.PoolClient>;
   /**
    * Opens a new connection, begins a transaction, and sets the user context.
    *
@@ -132,17 +141,6 @@ export interface IDBConnection {
    * @memberof IDBConnection
    */
   rollback: () => Promise<void>;
-  /**
-   * Performs a query against this connection, returning the results.
-   *
-   * @param {string} text SQL text
-   * @param {any[]} [values] SQL values array (optional)
-   * @return {*}  {(Promise<QueryResult<any>>)}
-   * @throws If the connection is not open.
-   * @deprecated Prefer using `.sql` (pass entire statement object) or `.knex` (pass knex query builder object)
-   * @memberof IDBConnection
-   */
-  query: <T extends pg.QueryResultRow = any>(text: string, values?: any[]) => Promise<pg.QueryResult<T>>;
   /**
    * Performs a query against this connection, returning the results.
    *
@@ -236,6 +234,21 @@ export const getDBConnection = function (keycloakToken?: KeycloakUserInformation
   let _systemUserId: number | null = null;
   const _token = keycloakToken;
 
+  /**
+   * Get a new pg client.
+   *
+   * @return {*}
+   */
+  const _getClient = async () => {
+    const pool = getDBPool();
+
+    if (!pool) {
+      throw Error('DBPool is not initialized');
+    }
+
+    return pool.connect();
+  };
+
   /**
    * Opens a new connection, begins a transaction, and sets the user context.
    *
@@ -553,8 +566,8 @@ export const getDBConnection = function (keycloakToken?: KeycloakUserInformation
   };
 
   return {
+    getClient: asyncErrorWrapper(_getClient),
     open: asyncErrorWrapper(_open),
-    query: asyncErrorWrapper(_query),
     sql: asyncErrorWrapper(_sql),
     knex: asyncErrorWrapper(_knex),
     release: syncErrorWrapper(_release),

api/src/models/animal-view.ts

@@ -1,8 +1,39 @@
 export interface IAnimalAdvancedFilters {
+  /**
+   * Filter results by keyword.
+   *
+   * @type {string}
+   * @memberof IAnimalAdvancedFilters
+   */
   keyword?: string;
+  /**
+   * Filter results by ITIS TSNs.
+   *
+   * @type {number[]}
+   * @memberof IAnimalAdvancedFilters
+   */
   itis_tsns?: number[];
+  /**
+   * Filter results by ITIS TSN.
+   *
+   * @type {number}
+   * @memberof IAnimalAdvancedFilters
+   */
   itis_tsn?: number;
+  /**
+   * Filter results by system user id (not necessarily the user making the request).
+   *
+   * @type {number}
+   * @memberof IAnimalAdvancedFilters
+   */
   system_user_id?: number;
+  /**
+   * Filter results by survey ids
+   *
+   * @type {number[]}
+   * @memberof IAnimalAdvancedFilters
+   */
+  survey_ids?: number[];
 }
 
 export interface ISex {

api/src/models/telemetry-view.ts

@@ -1,6 +1,37 @@
 export interface IAllTelemetryAdvancedFilters {
+  /**
+   * Filter results by keyword.
+   *
+   * @type {string}
+   * @memberof IAnimalAdvancedFilters
+   */
   keyword?: string;
+  /**
+   * Filter results by ITIS TSNs.
+   *
+   * @type {number[]}
+   * @memberof IAnimalAdvancedFilters
+   */
   itis_tsns?: number[];
+  /**
+   * Filter results by ITIS TSN.
+   *
+   * @type {number}
+   * @memberof IAnimalAdvancedFilters
+   */
   itis_tsn?: number;
+  /**
+   * Filter results by system user id (not necessarily the user making the request).
+   *
+   * @type {number}
+   * @memberof IAnimalAdvancedFilters
+   */
   system_user_id?: number;
+  /**
+   * Filter results by survey ids
+   *
+   * @type {number[]}
+   * @memberof IAnimalAdvancedFilters
+   */
+  survey_ids?: number[];
 }

api/src/paths/project/{projectId}/attachments/report/upload.test.ts

@@ -46,8 +46,6 @@ describe('uploadMedia', () => {
       }
     });
 
-    sinon.stub(file_utils, 'scanFileForVirus').resolves(true);
-
     const expectedError = new Error('cannot process request');
     sinon.stub(AttachmentService.prototype, 'upsertProjectReportAttachment').rejects(expectedError);
 
@@ -69,7 +67,6 @@ describe('uploadMedia', () => {
       }
     });
 
-    sinon.stub(file_utils, 'scanFileForVirus').resolves(true);
     sinon.stub(file_utils, 'uploadFileToS3').resolves();
 
     const expectedResponse = { attachmentId: 1, revision_count: 1 };

api/src/paths/project/{projectId}/attachments/report/upload.ts

@@ -2,11 +2,10 @@ import { RequestHandler } from 'express';
 import { Operation } from 'express-openapi';
 import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../constants/roles';
 import { getDBConnection } from '../../../../../database/db';
-import { HTTP400 } from '../../../../../errors/http-error';
 import { fileSchema } from '../../../../../openapi/schemas/file';
 import { authorizeRequestHandler } from '../../../../../request-handlers/security/authorization';
 import { AttachmentService } from '../../../../../services/attachment-service';
-import { scanFileForVirus, uploadFileToS3 } from '../../../../../utils/file-utils';
+import { uploadFileToS3 } from '../../../../../utils/file-utils';
 import { getLogger } from '../../../../../utils/logger';
 import { getFileFromRequest } from '../../../../../utils/request';
 
@@ -158,13 +157,6 @@ export function uploadMedia(): RequestHandler {
     try {
       await connection.open();
 
-      // Scan file for viruses using ClamAV
-      const virusScanResult = await scanFileForVirus(rawMediaFile);
-
-      if (!virusScanResult) {
-        throw new HTTP400('Malicious content detected, upload cancelled');
-      }
-
       const attachmentService = new AttachmentService(connection);
 
       //Upsert a report attachment