From 7371f5ed6b21ba5e87e2acea9ad3b43e58bdeea2 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Fri, 15 Dec 2023 16:39:54 -0800 Subject: [PATCH 1/3] Fix geojson zod schema, update publish classes to match latest version expected by BioHub. --- api/src/database/db.ts | 2 +- api/src/models/biohub-create.ts | 42 ++++++++++++++++--- .../survey-location-repository.ts | 2 +- api/src/services/platform-service.ts | 16 +++---- api/src/zod-schema/geoJsonZodSchema.ts | 8 ++-- 5 files changed, 52 insertions(+), 18 deletions(-) diff --git a/api/src/database/db.ts b/api/src/database/db.ts index 396daf5b1e..335e425592 100644 --- a/api/src/database/db.ts +++ b/api/src/database/db.ts @@ -354,7 +354,7 @@ export const getDBConnection = function (keycloakToken: KeycloakUserInformation) const zodEnd = Date.now(); defaultLog.silly({ - label: '_sq + zod', + label: '_sql + zod', message: sqlStatement.text, queryExecutionTime: queryEnd - queryStart, zodExecutionTime: zodEnd - zodStart diff --git a/api/src/models/biohub-create.ts b/api/src/models/biohub-create.ts index 2523b89f16..bc72dfc2e0 100644 --- a/api/src/models/biohub-create.ts +++ b/api/src/models/biohub-create.ts @@ -5,17 +5,25 @@ import { GetSurveyData } from './survey-view'; const defaultLog = getLogger('models/biohub-create'); +export interface BioHubSubmissionFeature { + id: string; + type: string; + properties: Record; + features: BioHubSubmissionFeature[]; +} + /** * Object to be sent to Biohub API for creating an observation. * * @export * @class PostSurveyObservationToBiohubObject + * @implements {BioHubSubmissionFeature} */ -export class PostSurveyObservationToBiohubObject { +export class PostSurveyObservationToBiohubObject implements BioHubSubmissionFeature { id: string; type: string; - properties: object; - features: []; + properties: Record; + features: BioHubSubmissionFeature[]; constructor(observationRecord: ObservationRecord) { defaultLog.debug({ label: 'PostSurveyObservationToBiohubObject', message: 'params', observationRecord }); @@ -43,11 +51,12 @@ export class PostSurveyObservationToBiohubObject { * * @export * @class PostSurveyToBiohubObject + * @implements {BioHubSubmissionFeature} */ -export class PostSurveyToBiohubObject { +export class PostSurveyToBiohubObject implements BioHubSubmissionFeature { id: string; type: string; - properties: object; + properties: Record; features: PostSurveyObservationToBiohubObject[]; constructor( @@ -75,6 +84,29 @@ export class PostSurveyToBiohubObject { } } +export class PostSurveySubmissionToBioHubObject { + id: string; + name: string; + description: string; + features: BioHubSubmissionFeature[]; + + constructor( + surveyData: GetSurveyData, + observationRecords: ObservationRecord[], + surveyGeometry: FeatureCollection, + additionalInformation?: string + ) { + defaultLog.debug({ label: 'PostSurveySubmissionToBioHubObject' }); + + this.id = surveyData.uuid; + this.name = surveyData.survey_name; + this.description = 'A Temp Description'; // TODO replace with data entered in survey publish dialog + this.features = [ + new PostSurveyToBiohubObject(surveyData, observationRecords, surveyGeometry, additionalInformation) + ]; + } +} + export enum BiohubFeatureType { DATASET = 'dataset', OBSERVATION = 'observation' diff --git a/api/src/repositories/survey-location-repository.ts b/api/src/repositories/survey-location-repository.ts index 6b9b74f7f5..c8f4284324 100644 --- a/api/src/repositories/survey-location-repository.ts +++ b/api/src/repositories/survey-location-repository.ts @@ -12,7 +12,7 @@ export const SurveyLocationRecord = z.object({ description: z.string(), geometry: z.record(z.any()).nullable(), geography: z.string(), - geojson: GeoJSONFeatureZodSchema, + geojson: z.array(GeoJSONFeatureZodSchema), revision_count: z.number() }); diff --git a/api/src/services/platform-service.ts b/api/src/services/platform-service.ts index 8378ef5172..cd9569213a 100644 --- a/api/src/services/platform-service.ts +++ b/api/src/services/platform-service.ts @@ -6,7 +6,7 @@ import { URL } from 'url'; import { v4 as uuidv4 } from 'uuid'; import { IDBConnection } from '../database/db'; import { ApiError, ApiErrorType, ApiGeneralError } from '../errors/api-error'; -import { PostSurveyToBiohubObject } from '../models/biohub-create'; +import { PostSurveySubmissionToBioHubObject } from '../models/biohub-create'; import { IProjectAttachment, IProjectReportAttachment, @@ -264,10 +264,9 @@ export class PlatformService extends DBService { /** * Submit survey ID to BioHub. * - * @param {string} surveyUUID * @param {number} surveyId * @param {{ additionalInformation: string }} data - * @return {*} {Promise<{ queue_id: number }>} + * @return {*} {Promise<{ submission_id: number }>} * @memberof PlatformService */ async submitSurveyToBioHub( @@ -316,10 +315,13 @@ export class PlatformService extends DBService { * * @param {number} surveyId * @param {string} additionalInformation - * @return {*} {Promise} + * @return {*} {Promise} * @memberof PlatformService */ - async generateSurveyDataPackage(surveyId: number, additionalInformation: string): Promise { + async generateSurveyDataPackage( + surveyId: number, + additionalInformation: string + ): Promise { const observationService = new ObservationService(this.connection); const surveyService = new SurveyService(this.connection); @@ -329,10 +331,10 @@ export class PlatformService extends DBService { const geometryFeatureCollection: FeatureCollection = { type: 'FeatureCollection', - features: surveyLocation.flatMap((location) => location.geojson as Feature) + features: surveyLocation.flatMap((location) => location.geojson as Feature[]) }; - const surveyDataPackage = new PostSurveyToBiohubObject( + const surveyDataPackage = new PostSurveySubmissionToBioHubObject( survey, surveyObservations, geometryFeatureCollection, diff --git a/api/src/zod-schema/geoJsonZodSchema.ts b/api/src/zod-schema/geoJsonZodSchema.ts index 44408d817b..101f9ba865 100644 --- a/api/src/zod-schema/geoJsonZodSchema.ts +++ b/api/src/zod-schema/geoJsonZodSchema.ts @@ -38,20 +38,20 @@ export const GeoJSONMultiPolygonZodSchema = z.object({ export const GeoJSONGeometryCollectionZodSchema = z.object({ type: z.enum(['GeometryCollection']), - geometries: z.array(z.object({})), + geometries: z.array(z.record(z.string(), z.any())), bbox: z.array(z.number()).min(4).optional() }); export const GeoJSONFeatureZodSchema = z.object({ type: z.enum(['Feature']), id: z.union([z.number(), z.string()]).optional(), - properties: z.object({}), - geometry: z.object({}), + properties: z.record(z.string(), z.any()), + geometry: z.record(z.string(), z.any()), bbox: z.array(z.number()).min(4).optional() }); export const GeoJSONFeatureCollectionZodSchema = z.object({ type: z.enum(['FeatureCollection']), - features: z.array(z.object({})), + features: z.array(z.record(z.string(), z.any())), bbox: z.array(z.number()).min(4).optional() }); From b9cb940688f9bc4ca54dd7d093b69da61ac00ea4 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Mon, 18 Dec 2023 10:26:32 -0800 Subject: [PATCH 2/3] Add/fix unit tests --- api/src/models/biohub-create.test.ts | 90 ++++++++++++++++++----- api/src/services/platform-service.test.ts | 61 ++++++++------- 2 files changed, 104 insertions(+), 47 deletions(-) diff --git a/api/src/models/biohub-create.test.ts b/api/src/models/biohub-create.test.ts index a68d2149d7..ab85928da6 100644 --- a/api/src/models/biohub-create.test.ts +++ b/api/src/models/biohub-create.test.ts @@ -1,7 +1,12 @@ import { expect } from 'chai'; +import { FeatureCollection } from 'geojson'; import { describe } from 'mocha'; import { ObservationRecord } from '../repositories/observation-repository'; -import { PostSurveyObservationToBiohubObject, PostSurveyToBiohubObject } from './biohub-create'; +import { + PostSurveyObservationToBiohubObject, + PostSurveySubmissionToBioHubObject, + PostSurveyToBiohubObject +} from './biohub-create'; import { GetSurveyData } from './survey-view'; describe('PostSurveyObservationToBiohubObject', () => { @@ -127,25 +132,70 @@ describe('PostSurveyToBiohubObject', () => { }); it('sets features', () => { - expect(data.features).to.eql([ - { - id: '1', - type: 'observation', - properties: { - survey_id: 1, - taxonomy: 1, - survey_sample_site_id: 1, - survey_sample_method_id: 1, - survey_sample_period_id: 1, - latitude: 1, - longitude: 1, - count: 1, - observation_time: 'observation_time', - observation_date: 'observation_date' - }, - features: [] - } - ]); + expect(data.features).to.eql([new PostSurveyObservationToBiohubObject(observation_obj)]); + }); + }); +}); + +describe('PostSurveySubmissionToBioHubObject', () => { + describe('All values provided', () => { + let data: PostSurveySubmissionToBioHubObject; + + const observation_obj: ObservationRecord[] = [ + { + survey_observation_id: 1, + survey_id: 1, + wldtaxonomic_units_id: 1, + survey_sample_site_id: 1, + survey_sample_method_id: 1, + survey_sample_period_id: 1, + latitude: 1, + longitude: 1, + count: 1, + observation_time: 'observation_time', + observation_date: 'observation_date', + create_date: 'create_date', + create_user: 1, + update_date: 'update_date', + update_user: 1, + revision_count: 1 + } + ]; + + const survey_obj: GetSurveyData = { + id: 1, + uuid: '1', + project_id: 1, + survey_name: 'survey_name', + start_date: 'start_date', + end_date: 'end_date', + survey_types: [9], + revision_count: 1 + }; + + const survey_geometry: FeatureCollection = { + type: 'FeatureCollection', + features: [] + }; + + before(() => { + data = new PostSurveySubmissionToBioHubObject(survey_obj, observation_obj, survey_geometry); + }); + + it('sets id', () => { + expect(data.id).to.equal('1'); + }); + + it('sets name', () => { + expect(data.name).to.equal('survey_name'); + }); + + it('sets description', () => { + expect(data.description).to.equal('A Temp Description'); + }); + + it('sets features', () => { + expect(data.features).to.eql([new PostSurveyToBiohubObject(survey_obj, observation_obj, survey_geometry)]); }); }); }); diff --git a/api/src/services/platform-service.test.ts b/api/src/services/platform-service.test.ts index 9bb5c6323a..fcbfb67175 100644 --- a/api/src/services/platform-service.test.ts +++ b/api/src/services/platform-service.test.ts @@ -351,38 +351,45 @@ describe('PlatformService', () => { expect(getSurveyLocationsDataStub).to.have.been.calledOnceWith(1); expect(response).to.eql({ id: '1', - type: 'dataset', - properties: { - additional_information: 'test', - survey_id: undefined, - project_id: undefined, - name: undefined, - start_date: undefined, - end_date: undefined, - survey_types: undefined, - revision_count: undefined, - geometry: { - type: 'FeatureCollection', - features: [] - } - }, + name: undefined, + description: 'A Temp Description', features: [ { - id: '2', - type: 'observation', + id: '1', + type: 'dataset', properties: { + additional_information: 'test', survey_id: undefined, - taxonomy: undefined, - survey_sample_site_id: null, - survey_sample_method_id: null, - survey_sample_period_id: null, - latitude: undefined, - longitude: undefined, - count: undefined, - observation_time: undefined, - observation_date: undefined + project_id: undefined, + name: undefined, + start_date: undefined, + end_date: undefined, + survey_types: undefined, + revision_count: undefined, + geometry: { + type: 'FeatureCollection', + features: [] + } }, - features: [] + features: [ + { + id: '2', + type: 'observation', + properties: { + survey_id: undefined, + taxonomy: undefined, + survey_sample_site_id: null, + survey_sample_method_id: null, + survey_sample_period_id: null, + latitude: undefined, + longitude: undefined, + count: undefined, + observation_time: undefined, + observation_date: undefined + }, + features: [] + } + ] } ] }); From 1610ea45a995026c6b663c10496578f25dead696 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Mon, 18 Dec 2023 13:11:52 -0800 Subject: [PATCH 3/3] Add observation geometry creation to biohub submit class. Assign additional information to description field. --- api/src/models/biohub-create.test.ts | 34 ++++++++++++++++------- api/src/models/biohub-create.ts | 34 ++++++++++++++--------- api/src/paths/publish/survey.ts | 5 ++-- api/src/services/platform-service.test.ts | 8 +++--- 4 files changed, 52 insertions(+), 29 deletions(-) diff --git a/api/src/models/biohub-create.test.ts b/api/src/models/biohub-create.test.ts index ab85928da6..b33036abc4 100644 --- a/api/src/models/biohub-create.test.ts +++ b/api/src/models/biohub-create.test.ts @@ -55,7 +55,20 @@ describe('PostSurveyObservationToBiohubObject', () => { longitude: 1, count: 1, observation_time: 'observation_time', - observation_date: 'observation_date' + observation_date: 'observation_date', + geometry: { + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + geometry: { + type: 'Point', + coordinates: [1, 1] + }, + properties: {} + } + ] + } }); }); }); @@ -101,12 +114,7 @@ describe('PostSurveyToBiohubObject', () => { } as GetSurveyData; before(() => { - data = new PostSurveyToBiohubObject( - survey_obj, - [observation_obj], - { type: 'FeatureCollection', features: [] }, - 'additionalInformation' - ); + data = new PostSurveyToBiohubObject(survey_obj, [observation_obj], { type: 'FeatureCollection', features: [] }); }); it('sets id', () => { @@ -119,7 +127,6 @@ describe('PostSurveyToBiohubObject', () => { it('sets properties', () => { expect(data.properties).to.eql({ - additional_information: 'additionalInformation', survey_id: 1, project_id: 1, name: 'survey_name', @@ -178,8 +185,15 @@ describe('PostSurveySubmissionToBioHubObject', () => { features: [] }; + const additionalInformation = 'A description of the submission'; + before(() => { - data = new PostSurveySubmissionToBioHubObject(survey_obj, observation_obj, survey_geometry); + data = new PostSurveySubmissionToBioHubObject( + survey_obj, + observation_obj, + survey_geometry, + additionalInformation + ); }); it('sets id', () => { @@ -191,7 +205,7 @@ describe('PostSurveySubmissionToBioHubObject', () => { }); it('sets description', () => { - expect(data.description).to.equal('A Temp Description'); + expect(data.description).to.equal('A description of the submission'); }); it('sets features', () => { diff --git a/api/src/models/biohub-create.ts b/api/src/models/biohub-create.ts index bc72dfc2e0..8d3fa1fe3a 100644 --- a/api/src/models/biohub-create.ts +++ b/api/src/models/biohub-create.ts @@ -40,7 +40,23 @@ export class PostSurveyObservationToBiohubObject implements BioHubSubmissionFeat longitude: observationRecord.longitude, count: observationRecord.count, observation_time: observationRecord.observation_time, - observation_date: observationRecord.observation_date + observation_date: observationRecord.observation_date, + geometry: { + type: 'FeatureCollection', + features: + observationRecord.longitude && observationRecord.latitude + ? [ + { + type: 'Feature', + geometry: { + type: 'Point', + coordinates: [observationRecord.longitude, observationRecord.latitude] + }, + properties: {} + } + ] + : [] + } }; this.features = []; } @@ -59,18 +75,12 @@ export class PostSurveyToBiohubObject implements BioHubSubmissionFeature { properties: Record; features: PostSurveyObservationToBiohubObject[]; - constructor( - surveyData: GetSurveyData, - observationRecords: ObservationRecord[], - surveyGeometry: FeatureCollection, - additionalInformation?: string - ) { + constructor(surveyData: GetSurveyData, observationRecords: ObservationRecord[], surveyGeometry: FeatureCollection) { defaultLog.debug({ label: 'PostSurveyToBiohubObject', message: 'params', surveyData }); this.id = surveyData.uuid; this.type = BiohubFeatureType.DATASET; this.properties = { - additional_information: additionalInformation ?? null, survey_id: surveyData.id, project_id: surveyData.project_id, name: surveyData.survey_name, @@ -94,16 +104,14 @@ export class PostSurveySubmissionToBioHubObject { surveyData: GetSurveyData, observationRecords: ObservationRecord[], surveyGeometry: FeatureCollection, - additionalInformation?: string + additionalInformation: string ) { defaultLog.debug({ label: 'PostSurveySubmissionToBioHubObject' }); this.id = surveyData.uuid; this.name = surveyData.survey_name; - this.description = 'A Temp Description'; // TODO replace with data entered in survey publish dialog - this.features = [ - new PostSurveyToBiohubObject(surveyData, observationRecords, surveyGeometry, additionalInformation) - ]; + this.description = additionalInformation; + this.features = [new PostSurveyToBiohubObject(surveyData, observationRecords, surveyGeometry)]; } } diff --git a/api/src/paths/publish/survey.ts b/api/src/paths/publish/survey.ts index 6e5903b7aa..75929eedbb 100644 --- a/api/src/paths/publish/survey.ts +++ b/api/src/paths/publish/survey.ts @@ -44,10 +44,11 @@ POST.apiDoc = { required: ['surveyId', 'data'], properties: { surveyId: { - type: 'number' + type: 'integer', + minimum: 1 }, data: { - description: 'All survey data to upload', + description: 'Additional data to include in the submission to BioHub', type: 'object', required: ['additionalInformation'], properties: { diff --git a/api/src/services/platform-service.test.ts b/api/src/services/platform-service.test.ts index fcbfb67175..2400ef8a56 100644 --- a/api/src/services/platform-service.test.ts +++ b/api/src/services/platform-service.test.ts @@ -344,7 +344,7 @@ describe('PlatformService', () => { .stub(SurveyService.prototype, 'getSurveyLocationsData') .resolves([] as any); - const response = await platformService.generateSurveyDataPackage(1, 'test'); + const response = await platformService.generateSurveyDataPackage(1, 'additional information'); expect(getSurveyDataStub).to.have.been.calledOnceWith(1); expect(getSurveyObservationsWithSupplementaryDataStub).to.have.been.calledOnceWith(1); @@ -352,13 +352,12 @@ describe('PlatformService', () => { expect(response).to.eql({ id: '1', name: undefined, - description: 'A Temp Description', + description: 'additional information', features: [ { id: '1', type: 'dataset', properties: { - additional_information: 'test', survey_id: undefined, project_id: undefined, name: undefined, @@ -385,7 +384,8 @@ describe('PlatformService', () => { longitude: undefined, count: undefined, observation_time: undefined, - observation_date: undefined + observation_date: undefined, + geometry: { type: 'FeatureCollection', features: [] } }, features: [] }