diff --git a/hrm-domain/hrm-core/case/adminCaseRoutesV0.ts b/hrm-domain/hrm-core/case/adminCaseRoutesV0.ts new file mode 100644 index 000000000..da6302bb3 --- /dev/null +++ b/hrm-domain/hrm-core/case/adminCaseRoutesV0.ts @@ -0,0 +1,46 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import type { Request, Response, NextFunction } from 'express'; +import { isErr, mapHTTPError } from '@tech-matters/types'; +import { SafeRouter, publicEndpoint } from '../permissions'; +import { reindexCases } from './caseReindexService'; + +const adminContactsRouter = SafeRouter(); + +// admin POST endpoint to reindex contacts. req body has accountSid, dateFrom, dateTo +adminContactsRouter.post( + '/reindex', + publicEndpoint, + async (req: Request, res: Response, next: NextFunction) => { + const { hrmAccountId } = req; + const { dateFrom, dateTo } = req.body; + + const result = await reindexCases(hrmAccountId, dateFrom, dateTo); + + if (isErr(result)) { + return next( + mapHTTPError(result, { + InvalidParameterError: 400, + }), + ); + } + + res.json(result.data); + }, +); + +export default adminContactsRouter.expressRouter; diff --git a/hrm-domain/hrm-core/case/caseReindexService.ts b/hrm-domain/hrm-core/case/caseReindexService.ts new file mode 100644 index 000000000..66fd34ed6 --- /dev/null +++ b/hrm-domain/hrm-core/case/caseReindexService.ts @@ -0,0 +1,74 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import { HrmAccountId, newErr, newOkFromData } from '@tech-matters/types'; +import { CaseService, searchCases } from './caseService'; +import { publishCaseToSearchIndex } from '../jobs/search/publishToSearchIndex'; +import { maxPermissions } from '../permissions'; +import { + AsyncProcessor, + SearchFunction, + processInBatch, +} from '@tech-matters/batch-processing'; +import formatISO from 'date-fns/formatISO'; + +export const reindexCases = async ( + accountSid: HrmAccountId, + dateFrom: string, + dateTo: string, +) => { + try { + const filters = { + updatedAt: { + from: formatISO(new Date(dateFrom)), + to: formatISO(new Date(dateTo)), + }, + }; + + const searchFunction: SearchFunction = async limitAndOffset => { + const res = await searchCases( + accountSid, + { + limit: limitAndOffset.limit.toString(), + offset: limitAndOffset.offset.toString(), + }, + {}, + { filters }, + maxPermissions, + ); + return { records: res.cases as CaseService[], count: res.count }; + }; + + const asyncProcessor: AsyncProcessor = async casesResult => { + const promises = casesResult.records.map(caseObj => { + return publishCaseToSearchIndex({ + accountSid, + case: caseObj, + operation: 'index', + }); + }); + + await Promise.all(promises); + }; + + await processInBatch(searchFunction, asyncProcessor); + + return newOkFromData('Successfully indexed contacts'); + } catch (error) { + console.error('Error reindexing contacts', error); + return newErr({ error, message: 'Error reindexing contacts' }); + } +}; diff --git a/hrm-domain/hrm-core/contact/adminContactRoutesV0.ts b/hrm-domain/hrm-core/contact/adminContactRoutesV0.ts new file mode 100644 index 000000000..2c81cf9f1 --- /dev/null +++ b/hrm-domain/hrm-core/contact/adminContactRoutesV0.ts @@ -0,0 +1,46 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import type { Request, Response, NextFunction } from 'express'; +import { isErr, mapHTTPError } from '@tech-matters/types'; +import { SafeRouter, publicEndpoint } from '../permissions'; +import { reindexContacts } from './contactsReindexService'; + +const adminContactsRouter = SafeRouter(); + +// admin POST endpoint to reindex contacts. req body has accountSid, dateFrom, dateTo +adminContactsRouter.post( + '/reindex', + publicEndpoint, + async (req: Request, res: Response, next: NextFunction) => { + const { hrmAccountId } = req; + const { dateFrom, dateTo } = req.body; + + const result = await reindexContacts(hrmAccountId, dateFrom, dateTo); + + if (isErr(result)) { + return next( + mapHTTPError(result, { + InvalidParameterError: 400, + }), + ); + } + + res.json(result.data); + }, +); + +export default adminContactsRouter.expressRouter; diff --git a/hrm-domain/hrm-core/contact/contactsReindexService.ts b/hrm-domain/hrm-core/contact/contactsReindexService.ts new file mode 100644 index 000000000..6e4346f52 --- /dev/null +++ b/hrm-domain/hrm-core/contact/contactsReindexService.ts @@ -0,0 +1,67 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import { HrmAccountId, newErr, newOkFromData } from '@tech-matters/types'; +import { Contact, searchContacts } from './contactService'; +import { publishContactToSearchIndex } from '../jobs/search/publishToSearchIndex'; +import { maxPermissions } from '../permissions'; +import { + AsyncProcessor, + SearchFunction, + processInBatch, +} from '@tech-matters/batch-processing'; + +export const reindexContacts = async ( + accountSid: HrmAccountId, + dateFrom: string, + dateTo: string, +) => { + try { + const searchParameters = { + dateFrom, + dateTo, + }; + + const searchFunction: SearchFunction = async limitAndOffset => { + const res = await searchContacts( + accountSid, + searchParameters, + limitAndOffset, + maxPermissions, + ); + return { records: res.contacts, count: res.count }; + }; + + const asyncProcessor: AsyncProcessor = async contactsResult => { + const promises = contactsResult.records.map(contact => { + return publishContactToSearchIndex({ + accountSid, + contact, + operation: 'index', + }); + }); + + await Promise.all(promises); + }; + + await processInBatch(searchFunction, asyncProcessor); + + return newOkFromData('Successfully indexed contacts'); + } catch (error) { + console.error('Error reindexing contacts', error); + return newErr({ error, message: 'Error reindexing contacts' }); + } +}; diff --git a/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts index 04a8af6c7..af9975e10 100644 --- a/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts +++ b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts @@ -37,7 +37,6 @@ const publishToSearchIndex = async ({ JSON.stringify(message), ); const queueUrl = await getSsmParameter(PENDING_INDEX_QUEUE_SSM_PATH); - return await sendSqsMessage({ queueUrl, message: JSON.stringify(message), diff --git a/hrm-domain/hrm-core/package.json b/hrm-domain/hrm-core/package.json index 055b26c0a..31adfbcdd 100644 --- a/hrm-domain/hrm-core/package.json +++ b/hrm-domain/hrm-core/package.json @@ -25,6 +25,7 @@ "dependencies": { "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", + "@tech-matters/batch-processing": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", "@tech-matters/s3-client": "^1.0.0", diff --git a/hrm-domain/hrm-core/permissions/index.ts b/hrm-domain/hrm-core/permissions/index.ts index b529cc93d..fc5fadc44 100644 --- a/hrm-domain/hrm-core/permissions/index.ts +++ b/hrm-domain/hrm-core/permissions/index.ts @@ -15,16 +15,16 @@ */ import { SafeRouterRequest } from './safe-router'; +import { rulesMap } from './rulesMap'; +import { type InitializedCan, initializeCanForRules } from './initializeCanForRules'; +import { type RulesFile } from './rulesMap'; +import type { Request, Response, NextFunction } from 'express'; +import type { TwilioUser } from '@tech-matters/twilio-worker-auth'; +import type { AccountSID } from '@tech-matters/types'; export { SafeRouter, publicEndpoint } from './safe-router'; -export { rulesMap } from './rulesMap'; -export { Actions, actionsMaps } from './actions'; - -import { InitializedCan, initializeCanForRules } from './initializeCanForRules'; -import { RulesFile } from './rulesMap'; -import type { Request, Response, NextFunction } from 'express'; -import { TwilioUser } from '@tech-matters/twilio-worker-auth'; -import { AccountSID } from '@tech-matters/types'; +export { type Actions, actionsMaps } from './actions'; +export { rulesMap }; declare global { namespace Express { @@ -72,3 +72,18 @@ export const setupPermissions = export type RequestWithPermissions = SafeRouterRequest & { can: InitializedCan; }; + +export const maxPermissions: { + user: TwilioUser; + can: () => boolean; + permissions: (typeof rulesMap)[keyof typeof rulesMap]; +} = { + can: () => true, + user: { + accountSid: 'ACxxx', + workerSid: 'WKxxx', + roles: ['supervisor'], + isSupervisor: true, + }, + permissions: rulesMap.open, +}; diff --git a/hrm-domain/hrm-core/routes.ts b/hrm-domain/hrm-core/routes.ts index 60a4adf71..ab8f93199 100644 --- a/hrm-domain/hrm-core/routes.ts +++ b/hrm-domain/hrm-core/routes.ts @@ -24,6 +24,8 @@ import referrals from './referral/referral-routes-v0'; import permissions from './permissions/permissions-routes-v0'; import profiles from './profile/profileRoutesV0'; import adminProfiles from './profile/adminProfileRoutesV0'; +import adminContacts from './contact/adminContactRoutesV0'; +import adminCases from './case/adminCaseRoutesV0'; import { Permissions } from './permissions'; export const HRM_ROUTES: { @@ -49,7 +51,11 @@ export const apiV0 = (rules: Permissions) => { export const ADMIN_ROUTES: { path: string; routerFactory: () => Router; -}[] = [{ path: '/profiles', routerFactory: () => adminProfiles }]; +}[] = [ + { path: '/profiles', routerFactory: () => adminProfiles }, + { path: '/contacts', routerFactory: () => adminContacts }, + { path: '/cases', routerFactory: () => adminCases }, +]; export const adminApiV0 = () => { const router: IRouter = Router(); diff --git a/hrm-domain/hrm-service/scripts/README.md b/hrm-domain/hrm-service/scripts/README.md index 699ca6b5e..8f99e0c86 100644 --- a/hrm-domain/hrm-service/scripts/README.md +++ b/hrm-domain/hrm-service/scripts/README.md @@ -22,6 +22,8 @@ admin-cli | ├── create: # Create a new profile flag | ├── edit: # Edit an existing profile flag | ├── delete: # Delete an existing profile flag +├── reindex +| ├── hrm: # Reindex contacts and cases based on date range ``` ### Usage diff --git a/hrm-domain/hrm-service/scripts/admin-commands/reindex.ts b/hrm-domain/hrm-service/scripts/admin-commands/reindex.ts new file mode 100644 index 000000000..c6846eb2a --- /dev/null +++ b/hrm-domain/hrm-service/scripts/admin-commands/reindex.ts @@ -0,0 +1,25 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +export const command = 'reindex '; +export const desc = 'admin endpoints for reindexing contacts and cases'; + +export const builder = function (yargs) { + return yargs.commandDir('reindex', { + exclude: /^(index|_)/, + extensions: ['ts'], + }); +}; diff --git a/hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts b/hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts new file mode 100644 index 000000000..24e48865b --- /dev/null +++ b/hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts @@ -0,0 +1,142 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import { getHRMInternalEndpointAccess } from '@tech-matters/service-discovery'; +// eslint-disable-next-line import/no-extraneous-dependencies +import { fetch } from 'undici'; +import { getAdminV0URL, staticKeyPattern } from '../../hrmInternalConfig'; + +export const command = 'hrm'; +export const describe = 'Reindex contacts and cases based on date range'; + +export const builder = { + co: { + alias: 'contacts', + describe: 'reindex contacts', + type: 'boolean', + default: false, + }, + ca: { + alias: 'cases', + describe: 'reindex cases', + type: 'boolean', + default: false, + }, + e: { + alias: 'environment', + describe: 'environment (e.g. development, staging, production)', + demandOption: true, + type: 'string', + }, + r: { + alias: 'region', + describe: 'region (e.g. us-east-1)', + demandOption: true, + type: 'string', + }, + a: { + alias: 'accountSid', + describe: 'account SID', + demandOption: true, + type: 'string', + }, + f: { + alias: 'dateFrom', + describe: 'start date (e.g. 2024-01-01)', + demandOption: true, + type: 'string', + }, + t: { + alias: 'dateTo', + describe: 'end date (e.g. 2024-12-31)', + demandOption: true, + type: 'string', + }, +}; + +export const handler = async ({ + region, + environment, + accountSid, + dateFrom, + dateTo, + contacts, + cases, +}) => { + try { + const timestamp = new Date().getTime(); + const assumeRoleParams = { + RoleArn: 'arn:aws:iam::712893914485:role/tf-admin', + RoleSessionName: `hrm-admin-cli-${timestamp}`, + }; + + const { authKey, internalResourcesUrl } = await getHRMInternalEndpointAccess({ + region, + environment, + staticKeyPattern, + assumeRoleParams, + }); + + if (!contacts && !cases) { + console.log( + 'Please specify contacts and/or cases option to reindex in your command', + ); + return; + } + + if (contacts) { + const url = getAdminV0URL(internalResourcesUrl, accountSid, '/contacts/reindex'); + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Basic ${authKey}`, + }, + body: JSON.stringify({ dateFrom, dateTo }), + }); + + if (!response.ok) { + console.error( + `Failed to submit request for reindexing contacts: ${response.statusText}`, + ); + } else { + console.log(`Reindexing contacts from ${dateFrom} to ${dateTo}...`); + } + } + + if (cases) { + const url = getAdminV0URL(internalResourcesUrl, accountSid, '/cases/reindexCases'); + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Basic ${authKey}`, + }, + body: JSON.stringify({ dateFrom, dateTo }), + }); + + if (!response.ok) { + console.error( + `Failed to submit request for reindexing cases: ${response.statusText}`, + ); + } else { + console.log(`Reindexing cases from ${dateFrom} to ${dateTo}...`); + } + } + } catch (err) { + console.error(err); + } +}; diff --git a/hrm-domain/hrm-service/tsconfig.build.json b/hrm-domain/hrm-service/tsconfig.build.json index 7694eb726..da06997ab 100644 --- a/hrm-domain/hrm-service/tsconfig.build.json +++ b/hrm-domain/hrm-service/tsconfig.build.json @@ -3,6 +3,8 @@ "extends": "./tsconfig.base.json", "files": [], "references": [ + { "path": "packages/types" }, + { "path": "packages/batch-processing" }, { "path": "packages/http" }, { "path": "packages/twilio-worker-auth" }, { "path": "packages/s3-client" }, @@ -11,7 +13,6 @@ { "path": "packages/sqs-client" }, { "path": "packages/elasticsearch-client" }, { "path": "packages/twilio-client" }, - { "path": "packages/types" }, { "path": "packages/service-discovery" }, { "path": "resources-domain/packages/resources-search-config" }, { "path": "resources-domain/resources-service" }, diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/context.ts b/hrm-domain/scheduled-tasks/hrm-data-pull/context.ts index 81409168a..05f988f6b 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/context.ts +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/context.ts @@ -15,9 +15,8 @@ */ import { getSsmParameter } from '@tech-matters/ssm-cache'; -import { rulesMap } from '@tech-matters/hrm-core/permissions/rulesMap'; import { HrmAccountId } from '@tech-matters/types'; -import { TwilioUser } from '@tech-matters/twilio-worker-auth'; +export { maxPermissions } from '@tech-matters/hrm-core/permissions/index'; // const sanitizeEnv = (env: string) => (env === 'local' ? 'development' : env); @@ -63,18 +62,3 @@ export const getContext = async (): Promise => { return context; }; - -export const maxPermissions: { - user: TwilioUser; - can: () => boolean; - permissions: (typeof rulesMap)[keyof typeof rulesMap]; -} = { - can: () => true, - user: { - accountSid: 'ACxxx', - workerSid: 'WKxxx', - roles: ['supervisor'], - isSupervisor: true, - }, - permissions: rulesMap.open, -}; diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/package.json b/hrm-domain/scheduled-tasks/hrm-data-pull/package.json index db9c6d72a..a2ee1beaa 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/package.json +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/package.json @@ -12,6 +12,7 @@ }, "dependencies" : { "@tech-matters/types": "^1.0.0", + "@tech-matters/batch-processing": "^1.0.0", "@tech-matters/twilio-worker-auth": "^1.0.0", "@tech-matters/s3-client": "^1.0.0", "@tech-matters/hrm-core": "^1.0.0", diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/pull-cases.ts b/hrm-domain/scheduled-tasks/hrm-data-pull/pull-cases.ts index 48521a517..3183045b4 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/pull-cases.ts +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/pull-cases.ts @@ -18,9 +18,8 @@ import format from 'date-fns/format'; import formatISO from 'date-fns/formatISO'; import { putS3Object } from '@tech-matters/s3-client'; import * as caseApi from '@tech-matters/hrm-core/case/caseService'; - +import { autoPaginate } from '@tech-matters/batch-processing'; import { getContext, maxPermissions } from './context'; -import { autoPaginate } from './auto-paginate'; import { parseISO } from 'date-fns'; const getSearchParams = (startDate: Date, endDate: Date) => ({ diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/pull-contacts.ts b/hrm-domain/scheduled-tasks/hrm-data-pull/pull-contacts.ts index 2843825e7..d9dcbe27a 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/pull-contacts.ts +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/pull-contacts.ts @@ -20,7 +20,7 @@ import { putS3Object } from '@tech-matters/s3-client'; import { getContext, maxPermissions } from './context'; import * as contactApi from '@tech-matters/hrm-core/contact/contactService'; -import { autoPaginate } from './auto-paginate'; +import { autoPaginate } from '@tech-matters/batch-processing'; import { parseISO } from 'date-fns'; const getSearchParams = (startDate: Date, endDate: Date) => ({ diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/pull-profiles.ts b/hrm-domain/scheduled-tasks/hrm-data-pull/pull-profiles.ts index 900e04a33..3dc1e3199 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/pull-profiles.ts +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/pull-profiles.ts @@ -23,9 +23,8 @@ import type { ProfileSection, ProfileWithRelationships, } from '@tech-matters/hrm-core/profile/profileDataAccess'; - +import { autoPaginate } from '@tech-matters/batch-processing'; import { getContext } from './context'; -import { autoPaginate } from './auto-paginate'; import { parseISO } from 'date-fns'; const getSearchParams = (startDate: Date, endDate: Date) => ({ diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/tests/unit/pull-cases.test.ts b/hrm-domain/scheduled-tasks/hrm-data-pull/tests/unit/pull-cases.test.ts index 876b3ec10..12a6c9e9c 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/tests/unit/pull-cases.test.ts +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/tests/unit/pull-cases.test.ts @@ -21,9 +21,9 @@ import addDays from 'date-fns/addDays'; import * as caseApi from '@tech-matters/hrm-core/case/caseService'; import * as context from '../../context'; -import { defaultLimitAndOffset } from '../../auto-paginate'; import { pullCases } from '../../pull-cases'; import { HrmAccountId } from '@tech-matters/types'; +import { defaultLimitAndOffset } from '@tech-matters/hrm-core/autoPaginate'; const { maxPermissions } = context; diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/tests/unit/pull-contacts.test.ts b/hrm-domain/scheduled-tasks/hrm-data-pull/tests/unit/pull-contacts.test.ts index 64c2b4170..efff35b95 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/tests/unit/pull-contacts.test.ts +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/tests/unit/pull-contacts.test.ts @@ -21,9 +21,9 @@ import addDays from 'date-fns/addDays'; import * as contactApi from '@tech-matters/hrm-core/contact/contactService'; import * as context from '../../context'; -import { defaultLimitAndOffset } from '../../auto-paginate'; import { pullContacts } from '../../pull-contacts'; import { HrmAccountId } from '@tech-matters/types'; +import { defaultLimitAndOffset } from '@tech-matters/hrm-core/autoPaginate'; const { maxPermissions } = context; diff --git a/package-lock.json b/package-lock.json index ef036281f..6f6b4fae2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -66,6 +66,7 @@ "version": "1.0.0", "license": "AGPL", "dependencies": { + "@tech-matters/batch-processing": "^1.0.0", "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", @@ -325,6 +326,7 @@ "name": "@tech-matters/hrm-data-pull", "version": "1.0.0", "dependencies": { + "@tech-matters/batch-processing": "^1.0.0", "@tech-matters/hrm-core": "^1.0.0", "@tech-matters/s3-client": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", @@ -3871,6 +3873,10 @@ "resolved": "lambdas/packages/alb-handler", "link": true }, + "node_modules/@tech-matters/batch-processing": { + "resolved": "packages/batch-processing", + "link": true + }, "node_modules/@tech-matters/case-status-transition": { "resolved": "hrm-domain/scheduled-tasks/case-status-transition", "link": true @@ -14389,6 +14395,11 @@ "resolved": "https://registry.npmjs.org/zstd-codec/-/zstd-codec-0.1.4.tgz", "integrity": "sha512-KYnWoFWgGtWyQEKNnUcb3u8ZtKO8dn5d8u+oGpxPlopqsPyv60U8suDyfk7Z7UtAO6Sk5i1aVcAs9RbaB1n36A==" }, + "packages/batch-processing": { + "name": "@tech-matters/batch-processing", + "version": "1.0.0", + "license": "AGPL" + }, "packages/elasticsearch-client": { "name": "@tech-matters/elasticsearch-client", "version": "1.0.0", @@ -17426,6 +17437,9 @@ "@types/aws-lambda": "^8.10.108" } }, + "@tech-matters/batch-processing": { + "version": "file:packages/batch-processing" + }, "@tech-matters/case-status-transition": { "version": "file:hrm-domain/scheduled-tasks/case-status-transition", "requires": { @@ -17475,6 +17489,7 @@ "@tech-matters/hrm-core": { "version": "file:hrm-domain/hrm-core", "requires": { + "@tech-matters/batch-processing": "^1.0.0", "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", @@ -17543,6 +17558,7 @@ "@tech-matters/hrm-data-pull": { "version": "file:hrm-domain/scheduled-tasks/hrm-data-pull", "requires": { + "@tech-matters/batch-processing": "^1.0.0", "@tech-matters/hrm-core": "^1.0.0", "@tech-matters/s3-client": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", diff --git a/packages/batch-processing/package.json b/packages/batch-processing/package.json new file mode 100644 index 000000000..7b092bf2b --- /dev/null +++ b/packages/batch-processing/package.json @@ -0,0 +1,11 @@ +{ + "name": "@tech-matters/batch-processing", + "version": "1.0.0", + "description": "", + "main": "dist/index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "AGPL" +} diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/auto-paginate.ts b/packages/batch-processing/src/autoPaginate.ts similarity index 73% rename from hrm-domain/scheduled-tasks/hrm-data-pull/auto-paginate.ts rename to packages/batch-processing/src/autoPaginate.ts index ac7347f26..456619b1d 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/auto-paginate.ts +++ b/packages/batch-processing/src/autoPaginate.ts @@ -29,22 +29,22 @@ type LimitAndOffset = { offset: number; }; -/** - * This function takes care of keep calling the search function - * until there's no more data to be fetched. It works by dynamically - * adjusting the 'offset' on each subsequent call. - * - * @param searchFunction function to perform search of cases or contacts with the provided limit & offset - * @returns cases[] or contacts[] - */ -export const autoPaginate = async ( - searchFunction: (limitAndOffset: LimitAndOffset) => Promise>, -): Promise => { - let items: T[] = []; +export type SearchFunction = ( + limitAndOffset: LimitAndOffset, +) => Promise>; + +export type AsyncProcessor = (result: SearchResult) => Promise; + +export const processInBatch = async ( + searchFunction: SearchFunction, + asyncProcessor: AsyncProcessor, +): Promise => { let hasMoreItems = true; let offset = Number(defaultLimitAndOffset.offset); const limit = Number(defaultLimitAndOffset.limit); + let processed = 0; + while (hasMoreItems) { /** * Updates 'limitAndOffset' param @@ -53,14 +53,36 @@ export const autoPaginate = async ( const searchResult = await searchFunction({ limit, offset }); const { count, records } = searchResult; - items = [...items, ...records]; - hasMoreItems = items.length < count; + await asyncProcessor(searchResult); + + processed += records.length; + hasMoreItems = processed < count; if (hasMoreItems) { offset += limit; } } +}; + +/** + * This function takes care of keep calling the search function + * until there's no more data to be fetched. It works by dynamically + * adjusting the 'offset' on each subsequent call. + * + * @param searchFunction function to perform search of cases or contacts with the provided limit & offset + * @returns cases[] or contacts[] + */ +export const autoPaginate = async ( + searchFunction: SearchFunction, +): Promise => { + let items: T[] = []; + + const asyncProcessor = async (result: SearchResult) => { + items.push(...result.records); + }; + + await processInBatch(searchFunction, asyncProcessor); return items; }; diff --git a/packages/batch-processing/src/index.ts b/packages/batch-processing/src/index.ts new file mode 100644 index 000000000..20f74fd3f --- /dev/null +++ b/packages/batch-processing/src/index.ts @@ -0,0 +1,17 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +export * from './autoPaginate'; diff --git a/packages/batch-processing/tsconfig.json b/packages/batch-processing/tsconfig.json new file mode 100644 index 000000000..43d19bed4 --- /dev/null +++ b/packages/batch-processing/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../tsconfig.packages-base.json", + "include": ["src/*.ts"], + "compilerOptions": { + "outDir": "./dist", + } +} \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index 804066797..07afb338c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,6 +4,7 @@ // The references must be in order. If package A is a dependency for package B, then A must be referenced before B. For more details see https://www.typescriptlang.org/docs/handbook/project-references.html "references": [ { "path": "packages/types" }, + { "path": "packages/batch-processing" }, { "path": "packages/testing" }, { "path": "packages/http" }, { "path": "packages/job-errors" },