diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/markings/import.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/markings/import.ts index 6a73a2ffbe..47711fe9f6 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/markings/import.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/markings/import.ts @@ -6,7 +6,7 @@ import { HTTP400 } from '../../../../../../../errors/http-error'; import { csvFileSchema } from '../../../../../../../openapi/schemas/file'; import { authorizeRequestHandler } from '../../../../../../../request-handlers/security/authorization'; import { importCSV } from '../../../../../../../services/import-services/csv-import-strategy'; -import { ImportMarkingsService } from '../../../../../../../services/import-services/marking/import-markings-service'; +import { ImportMarkingsStrategy } from '../../../../../../../services/import-services/marking/import-markings-strategy'; import { scanFileForVirus } from '../../../../../../../utils/file-utils'; import { getLogger } from '../../../../../../../utils/logger'; import { parseMulterFile } from '../../../../../../../utils/media/media-utils'; @@ -142,10 +142,10 @@ export function importCsv(): RequestHandler { throw new HTTP400('Malicious content detected, import cancelled.'); } - const importCsvMarkings = new ImportMarkingsService(connection, surveyId); + const importCsvMarkingsStrategy = new ImportMarkingsStrategy(connection, surveyId); // Pass CSV file and importer as dependencies - const markingsCreated = await importCSV(parseMulterFile(rawFile), importCsvMarkings); + const markingsCreated = await importCSV(parseMulterFile(rawFile), importCsvMarkingsStrategy); await connection.commit(); diff --git a/api/src/services/import-services/marking/import-markings-strategy.interface.ts b/api/src/services/import-services/marking/import-markings-strategy.interface.ts index 540fff2efa..0917f011b9 100644 --- a/api/src/services/import-services/marking/import-markings-strategy.interface.ts +++ b/api/src/services/import-services/marking/import-markings-strategy.interface.ts @@ -6,6 +6,9 @@ import { IAsSelectLookup } from '../../critterbase-service'; * * Note: This getter allows custom values to be injected for validation. * + * Note: This could be updated to transform the string values into the primary keys + * to prevent Critterbase from having to translate / patch in incomming bulk values. + * * @param {IAsSelectLookup[]} colours - Array of supported Critterbase colours * @returns {*} Custom Zod schema for CSV Markings */ @@ -56,7 +59,9 @@ export const getCsvMarkingSchema = ( code: z.ZodIssueCode.custom, message: 'No taxon body locations found for Critter' }); - } else if (!bodyLocations.filter((location) => location.value === schema.body_location).length) { + } else if ( + !bodyLocations.filter((location) => location.value.toLowerCase() === schema.body_location.toLowerCase()).length + ) { ctx.addIssue({ code: z.ZodIssueCode.custom, message: `Invalid body location for Critter. Allowed values: ${bodyLocations diff --git a/api/src/services/import-services/marking/import-markings-strategy.test.ts b/api/src/services/import-services/marking/import-markings-strategy.test.ts new file mode 100644 index 0000000000..94b614ee68 --- /dev/null +++ b/api/src/services/import-services/marking/import-markings-strategy.test.ts @@ -0,0 +1,140 @@ +import chai, { expect } from 'chai'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { getMockDBConnection } from '../../../__mocks__/db'; +import { IBulkCreateResponse, ICritterDetailed } from '../../critterbase-service'; +import { ImportMarkingsStrategy } from './import-markings-strategy'; +import { CsvMarking } from './import-markings-strategy.interface'; + +chai.use(sinonChai); + +describe.only('ImportMarkingsStrategy', () => { + describe('getTaxonBodyLocationsCritterIdMap', () => { + it('should return a critter_id mapping of body locations', async () => { + const mockDBConnection = getMockDBConnection(); + const strategy = new ImportMarkingsStrategy(mockDBConnection, 1); + + const taxonBodyLocationsStub = sinon.stub( + strategy.surveyCritterService.critterbaseService, + 'getTaxonBodyLocations' + ); + const mockBodyLocationsA = [ + { id: 'A', key: 'column', value: 'Right Ear' }, + { id: 'B', key: 'column', value: 'Antlers' } + ]; + + const mockBodyLocationsB = [ + { id: 'C', key: 'column', value: 'Nose' }, + { id: 'D', key: 'column', value: 'Tail' } + ]; + + taxonBodyLocationsStub.onCall(0).resolves(mockBodyLocationsA); + taxonBodyLocationsStub.onCall(1).resolves(mockBodyLocationsB); + + const critterMap = await strategy.getTaxonBodyLocationsCritterIdMap([ + { critter_id: 'ACRITTER', itis_tsn: 1 }, + { critter_id: 'BCRITTER', itis_tsn: 2 }, + { critter_id: 'CCRITTER', itis_tsn: 2 } + ] as ICritterDetailed[]); + + expect(taxonBodyLocationsStub).to.have.been.calledTwice; + expect(taxonBodyLocationsStub.getCall(0).args[0]).to.be.eql('1'); + expect(taxonBodyLocationsStub.getCall(1).args[0]).to.be.eql('2'); + expect(critterMap).to.be.deep.equal( + new Map([ + ['ACRITTER', mockBodyLocationsA], + ['BCRITTER', mockBodyLocationsB], + ['CCRITTER', mockBodyLocationsB] + ]) + ); + }); + }); + + describe('validateRows', () => { + it('should validate the rows successfully', async () => { + const mockDBConnection = getMockDBConnection(); + const strategy = new ImportMarkingsStrategy(mockDBConnection, 1); + + const mockCritterA = { + critter_id: '4df8fd4c-4d7b-4142-8f03-92d8bf52d8cb', + itis_tsn: 1, + captures: [ + { capture_id: 'e9087545-5b1f-4b86-bf1d-a3372a7b33c7', capture_date: '10-10-2024', capture_time: '10:10:10' } + ] + } as ICritterDetailed; + + const mockCritterB = { + critter_id: '4540d43a-7ced-4216-b49e-2a972d25dfdc', + itis_tsn: 1, + captures: [ + { capture_id: '21f3c699-9017-455b-bd7d-49110ca4b586', capture_date: '10-10-2024', capture_time: '10:10:10' } + ] + } as ICritterDetailed; + + const aliasStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterIdAliasMap'); + const colourStub = sinon.stub(strategy.surveyCritterService.critterbaseService, 'getColours'); + const markingTypeStub = sinon.stub(strategy.surveyCritterService.critterbaseService, 'getMarkingTypes'); + const taxonBodyLocationStub = sinon.stub(strategy, 'getTaxonBodyLocationsCritterIdMap'); + + aliasStub.resolves( + new Map([ + ['carl', mockCritterA], + ['carlita', mockCritterB] + ]) + ); + + colourStub.resolves([ + { id: 'A', key: 'colour', value: 'red' }, + { id: 'B', key: 'colour', value: 'blue' } + ]); + + markingTypeStub.resolves([ + { id: 'C', key: 'markingType', value: 'ear tag' }, + { id: 'D', key: 'markingType', value: 'nose band' } + ]); + + taxonBodyLocationStub.resolves( + new Map([ + ['4df8fd4c-4d7b-4142-8f03-92d8bf52d8cb', [{ id: 'D', key: 'bodylocation', value: 'ear' }]], + ['4540d43a-7ced-4216-b49e-2a972d25dfdc', [{ id: 'E', key: 'bodylocation', value: 'tail' }]] + ]) + ); + + const rows = [ + { + CAPTURE_DATE: '10-10-2024', + CAPTURE_TIME: '10:10:10', + ALIAS: 'carl', + BODY_LOCATION: 'Ear', + MARKING_TYPE: 'ear tag', + IDENTIFIER: 'identifier', + PRIMARY_COLOUR: 'Red', + SECONDARY_COLOUR: 'blue', + DESCRIPTION: 'comment' + } + ]; + + const validation = await strategy.validateRows(rows); + + if (!validation.success) { + expect.fail(); + } else { + } + }); + }); + describe('insert', () => { + it('should return the count of inserted markings', async () => { + const mockDBConnection = getMockDBConnection(); + const strategy = new ImportMarkingsStrategy(mockDBConnection, 1); + + const bulkCreateStub = sinon.stub(strategy.surveyCritterService.critterbaseService, 'bulkCreate'); + + bulkCreateStub.resolves({ created: { markings: 1 } } as IBulkCreateResponse); + + const data = await strategy.insert([{ critter_id: 'id' } as unknown as CsvMarking]); + + expect(bulkCreateStub).to.have.been.calledWith({ markings: [{ critter_id: 'id' }] }); + expect(data).to.be.eql(1); + }); + }); +}); diff --git a/api/src/services/import-services/marking/import-markings-strategy.ts b/api/src/services/import-services/marking/import-markings-strategy.ts index 11c400b652..e571c13471 100644 --- a/api/src/services/import-services/marking/import-markings-strategy.ts +++ b/api/src/services/import-services/marking/import-markings-strategy.ts @@ -1,5 +1,6 @@ import { z } from 'zod'; import { IDBConnection } from '../../../database/db'; +import { getLogger } from '../../../utils/logger'; import { CSV_COLUMN_ALIASES } from '../../../utils/xlsx-utils/column-aliases'; import { generateCellGetterFromColumnValidator } from '../../../utils/xlsx-utils/column-validator-utils'; import { IXLSXCSVValidator } from '../../../utils/xlsx-utils/worksheet-utils'; @@ -13,9 +14,11 @@ import { CsvMarking, getCsvMarkingSchema } from './import-markings-strategy.inte // TODO: Update all import services to use language ImportStrategy // TODO: Update CSVImportService interface -> CSVImportStrategy +const defaultLog = getLogger('services/import/import-markings-strategy'); + /** * - * @class ImportMarkingsService + * @class ImportMarkingsStrategy * @extends DBService * @see CSVImport * @@ -43,7 +46,7 @@ export class ImportMarkingsStrategy extends DBService implements CSVImportServic } satisfies IXLSXCSVValidator; /** - * Construct an instance of ImportMarkingsService. + * Construct an instance of ImportMarkingsStrategy. * * @param {IDBConnection} connection - DB connection * @param {string} surveyId @@ -105,11 +108,11 @@ export class ImportMarkingsStrategy extends DBService implements CSVImportServic const colours = await this.surveyCritterService.critterbaseService.getColours(); const markingTypes = await this.surveyCritterService.critterbaseService.getMarkingTypes(); - // Used to find critter_id -> body location map + // Used to find critter_id -> taxon body location [] map const rowCritters: ICritterDetailed[] = []; // Rows passed to validator - const rowsToValidate: Partial[] = []; + const rowsToValidate: Partial[] = []; for (const row of rows) { let critterId, captureId; @@ -142,10 +145,11 @@ export class ImportMarkingsStrategy extends DBService implements CSVImportServic comment: getCellValue(row, 'COMMENT') }); } - // Get the critter_id -> taxonBodyLocations[] mapping + // Get the critter_id -> taxonBodyLocations[] Map const critterBodyLocationsMap = await this.getTaxonBodyLocationsCritterIdMap(rowCritters); - // Generate the zod schema with injected lookup values + // Generate the zod schema with injected reference values + // This allows the zod schema to validate against Critterbase lookup values return z.array(getCsvMarkingSchema(colours, markingTypes, critterBodyLocationsMap)).safeParseAsync(rowsToValidate); } @@ -161,6 +165,8 @@ export class ImportMarkingsStrategy extends DBService implements CSVImportServic const response = await this.surveyCritterService.critterbaseService.bulkCreate(critterbasePayload); + defaultLog.debug({ label: 'import markings', markings, insertedCount: response.created.markings }); + return response.created.markings; } }