From c814b469ac69571ad41d347f35ed6dd3f4fbf211 Mon Sep 17 00:00:00 2001 From: Aleksandar Petkov Date: Tue, 10 Sep 2024 14:40:22 +0300 Subject: [PATCH] feat: Add endpoint to send newsletter consent email (#645) * feat: Add endpoint to send newsletter consent email * feat: Allow for dynamic setting for removing registeredUsers from mail list * chore: Add maximum number of retries when getting export status Needed to prevent potential infinite loop * chore: Make send-newsletter-consent endpoint protected * chore: Improvements * chore: Add content value dynamically based on fn call Not quite sure, what this value is for, but it is better to have it like this for now * fix: Linter error * chore: Improve TS definitions * chore: Code cleanup --- apps/api/src/common/mapChunk.ts | 17 +++ .../api/src/notifications/dto/massmail.dto.ts | 35 +++++ .../notifications/notifications.controller.ts | 13 +- .../notifications/notifications.service.ts | 103 ++++++++++++- .../notifications.interface.providers.ts | 21 +++ .../notifications.sendgrid.provider.ts | 142 +++++++++++++++++- .../providers/notifications.sendgrid.types.ts | 45 ++++++ 7 files changed, 372 insertions(+), 4 deletions(-) create mode 100644 apps/api/src/common/mapChunk.ts create mode 100644 apps/api/src/notifications/dto/massmail.dto.ts diff --git a/apps/api/src/common/mapChunk.ts b/apps/api/src/common/mapChunk.ts new file mode 100644 index 000000000..cd41b9bc6 --- /dev/null +++ b/apps/api/src/common/mapChunk.ts @@ -0,0 +1,17 @@ +/** + * Create a chunked array of new Map() + * @param map map to be chunked + * @param chunkSize The size of the chunk + * @returns Array chunk of new Map() + */ + +export function mapChunk>(map: T, chunkSize: number) { + return Array.from(map.entries()).reduce((chunk, curr, index) => { + const ch = Math.floor(index / chunkSize) + if (!chunk[ch]) { + chunk[ch] = new Map() as T + } + chunk[ch].set(curr[0], curr[1]) + return chunk + }, []) +} diff --git a/apps/api/src/notifications/dto/massmail.dto.ts b/apps/api/src/notifications/dto/massmail.dto.ts new file mode 100644 index 000000000..ead2c5395 --- /dev/null +++ b/apps/api/src/notifications/dto/massmail.dto.ts @@ -0,0 +1,35 @@ +import { ApiProperty } from '@nestjs/swagger' +import { Expose } from 'class-transformer' +import { IsDateString, IsNumber, IsOptional, IsString } from 'class-validator' + +export class MassMailDto { + @ApiProperty() + @Expose() + @IsString() + listId: string + + @ApiProperty() + @Expose() + @IsString() + templateId: string + + @ApiProperty() + @Expose() + @IsString() + @IsOptional() + subject: string + + //Sendgrid limits sending emails to 1000 at once. + @ApiProperty() + @Expose() + @IsNumber() + @IsOptional() + chunkSize = 1000 + + //Remove users registered after the dateThreshold from mail list + @ApiProperty() + @Expose() + @IsDateString() + @IsOptional() + dateThreshold: Date = new Date() +} diff --git a/apps/api/src/notifications/notifications.controller.ts b/apps/api/src/notifications/notifications.controller.ts index 72e15064c..505c66cbc 100644 --- a/apps/api/src/notifications/notifications.controller.ts +++ b/apps/api/src/notifications/notifications.controller.ts @@ -1,6 +1,6 @@ import { BadRequestException, Body, Controller, Get, Post } from '@nestjs/common' import { ApiTags } from '@nestjs/swagger' -import { AuthenticatedUser, Public } from 'nest-keycloak-connect' +import { AuthenticatedUser, Public, RoleMatchingMode, Roles } from 'nest-keycloak-connect' import { SendConfirmationDto, SubscribeDto, @@ -11,6 +11,8 @@ import { import { MarketingNotificationsService } from './notifications.service' import { KeycloakTokenParsed } from '../auth/keycloak' +import { MassMailDto } from './dto/massmail.dto' +import { RealmViewSupporters, ViewSupporters } from '@podkrepi-bg/podkrepi-types' @ApiTags('notifications') @Controller('notifications') @@ -63,4 +65,13 @@ export class MarketingNotificationsController { user.email || '', ) } + + @Post('/send-newsletter-consent') + @Roles({ + roles: [RealmViewSupporters.role, ViewSupporters.role], + mode: RoleMatchingMode.ANY, + }) + async sendMassMail(@Body() data: MassMailDto) { + return await this.marketingNotificationsService.sendConsentMail(data) + } } diff --git a/apps/api/src/notifications/notifications.service.ts b/apps/api/src/notifications/notifications.service.ts index 0c6ec988d..0972fed14 100644 --- a/apps/api/src/notifications/notifications.service.ts +++ b/apps/api/src/notifications/notifications.service.ts @@ -27,11 +27,28 @@ import { UnregisteredNotificationConsent, } from '@prisma/client' import { NotificationsProviderInterface } from './providers/notifications.interface.providers' -import { SendGridParams } from './providers/notifications.sendgrid.types' +import { ContactsResponse, SendGridParams } from './providers/notifications.sendgrid.types' import { DateTime } from 'luxon' import * as crypto from 'crypto' import { CampaignService } from '../campaign/campaign.service' import { KeycloakTokenParsed } from '../auth/keycloak' +import { MassMailDto } from './dto/massmail.dto' +import { randomUUID } from 'crypto' +import { mapChunk } from '../common/mapChunk' + +type UnregisteredInsert = { + id: string + email: string + consent: boolean +} + +type MailList = { + id: string + hash: string + registered: boolean +} + +export type ContactsMap = Map @Injectable() export class MarketingNotificationsService { @@ -492,4 +509,88 @@ export class MarketingNotificationsService { return minutesPassed <= period } + + private generateMapFromMailList(emailList: string[], contacts: ContactsMap): void { + for (const email of emailList) { + const id = randomUUID() + + contacts.set(email, { + id: id, + hash: this.generateHash(id), + registered: false, + }) + } + } + + private updateMailListMap( + regUser: Person[], + contacts: ContactsMap, + skipAfterDate: Date, + unregisteredConsent: UnregisteredNotificationConsent[], + ) { + for (const registeredUser of regUser) { + const createdAt = new Date(registeredUser.createdAt) + + // Remove email if it belongs to user created after the change has been deployed, as they had already decided + // whether to give consent or not. + if (contacts.get(registeredUser.email as string) && createdAt > skipAfterDate) { + Logger.debug(`Removing email ${registeredUser.email} from list`) + contacts.delete(registeredUser.email as string) + continue + } + //Update the value of this mail + contacts.set(registeredUser.email as string, { + id: registeredUser.id, + hash: this.generateHash(registeredUser.id), + registered: true, + }) + } + + Logger.debug('Removing emails in unregistered consent emails') + for (const consent of unregisteredConsent) { + if (contacts.has(consent.email)) { + Logger.debug(`Removing email ${consent.email}`) + contacts.delete(consent.email) + continue + } + } + } + + private async insertUnregisteredConsentFromContacts(contacts: ContactsMap) { + const emailsToAdd: UnregisteredInsert[] = [] + for (const [key, value] of contacts) { + if (value.registered) continue + emailsToAdd.push({ id: value.id, email: key, consent: false }) + } + + await this.prisma.unregisteredNotificationConsent.createMany({ + data: emailsToAdd, + }) + } + async sendConsentMail(data: MassMailDto) { + const contacts = await this.marketingNotificationsProvider.getContactsFromList(data) + + const sendList: ContactsMap = new Map() + const emailList = contacts.map((contact: ContactsResponse) => contact.email) + this.generateMapFromMailList(emailList, sendList) + const registeredMails = await this.prisma.person.findMany({ + where: { email: { in: emailList } }, + }) + + const unregisteredUsers = await this.prisma.unregisteredNotificationConsent.findMany() + + const skipUsersAfterDate = new Date(data.dateThreshold) + this.updateMailListMap(registeredMails, sendList, skipUsersAfterDate, unregisteredUsers) + + await this.insertUnregisteredConsentFromContacts(sendList) + + const contactsChunked = mapChunk(sendList, data.chunkSize) + Logger.debug(`Splitted email list into ${contactsChunked.length} chunk`) + await this.marketingNotificationsProvider.sendBulkEmail( + data, + contactsChunked, + 'Podkrepi.BG Newsletter Subscription Consent', + ) + return { contactCount: sendList.size } + } } diff --git a/apps/api/src/notifications/providers/notifications.interface.providers.ts b/apps/api/src/notifications/providers/notifications.interface.providers.ts index 4e6e6d28f..954e157a3 100644 --- a/apps/api/src/notifications/providers/notifications.interface.providers.ts +++ b/apps/api/src/notifications/providers/notifications.interface.providers.ts @@ -1,3 +1,7 @@ +import { MassMailDto } from '../dto/massmail.dto' +import { ContactsMap } from '../notifications.service' +import { PersonalizationData } from '@sendgrid/helpers/classes/personalization' + type NotificationsInterfaceParams = { CreateListParams: unknown UpdateListParams: unknown @@ -8,6 +12,7 @@ type NotificationsInterfaceParams = { RemoveFromUnsubscribedParams: unknown AddToUnsubscribedParams: unknown SendNotificationParams: unknown + GetContactsFromListParam: unknown // Responses CreateListRes: unknown @@ -19,6 +24,7 @@ type NotificationsInterfaceParams = { RemoveFromUnsubscribedRes: unknown AddToUnsubscribedRes: unknown SendNotificationRes: unknown + GetContactsFromListRes: unknown contactListsRes: unknown } @@ -36,5 +42,20 @@ export abstract class NotificationsProviderInterface< data: T['RemoveFromUnsubscribedParams'], ): Promise abstract sendNotification(data: T['SendNotificationParams']): Promise + abstract getContactsFromList( + data: T['GetContactsFromListParam'], + ): Promise + abstract prepareTemplatePersonalizations( + data: MassMailDto, + contacts: ContactsMap, + date?: Date, + ): PersonalizationData[] + + abstract sendBulkEmail( + data: MassMailDto, + contactsMap: ContactsMap[], + value: string, + timeout?: number, + ): Promise abstract getContactLists(): Promise } diff --git a/apps/api/src/notifications/providers/notifications.sendgrid.provider.ts b/apps/api/src/notifications/providers/notifications.sendgrid.provider.ts index 5778209c5..3b393de38 100644 --- a/apps/api/src/notifications/providers/notifications.sendgrid.provider.ts +++ b/apps/api/src/notifications/providers/notifications.sendgrid.provider.ts @@ -1,10 +1,28 @@ -import { Injectable, Logger } from '@nestjs/common' +import { + BadRequestException, + Injectable, + InternalServerErrorException, + Logger, + RequestTimeoutException, +} from '@nestjs/common' import { ConfigService } from '@nestjs/config' import sgClient from '@sendgrid/client' +import sgMail from '@sendgrid/mail' import { NotificationsProviderInterface } from './notifications.interface.providers' -import { ContactListRes, SGClientResponse, SendGridParams } from './notifications.sendgrid.types' +import { + ContactListRes, + ContactsFromListParams, + ContactsResponse, + SendGridExportStatusResponse, + SendGridParams, + SGClientResponse, +} from './notifications.sendgrid.types' import { ClientRequest } from '@sendgrid/client/src/request' import { DateTime } from 'luxon' +import { MassMailDto } from '../dto/massmail.dto' +import { ContactsMap } from '../notifications.service' +import { MailDataRequired } from '@sendgrid/mail' +import { PersonalizationData } from '@sendgrid/helpers/classes/personalization' @Injectable() export class SendGridNotificationsProvider @@ -20,6 +38,7 @@ export class SendGridNotificationsProvider if (apiKey) { sgClient.setApiKey(apiKey) + sgMail.setApiKey(apiKey) } else { Logger.warn('no apiKey for sendgrid, will not send notifications') } @@ -47,6 +66,76 @@ export class SendGridNotificationsProvider return listId } + private async createContactExport(listId: string) { + const request = { + url: `/v3/marketing/contacts/exports`, + method: 'POST', + body: { + list_ids: [listId], + file_type: 'json', + }, + } as ClientRequest + const [response] = await sgClient.request(request).catch((err) => { + throw new BadRequestException(`Couldn't create export. Error is ${err}`) + }) + return response.body as { id: string } + } + + private async getContactExportStatus(jobId: string) { + const request = { + url: `/v3/marketing/contacts/exports/${jobId}`, + method: 'GET', + } as ClientRequest + const [response] = await sgClient.request(request).catch((err) => { + throw new BadRequestException(`Couldn't create export. Error is ${err}`) + }) + return response.body as SendGridExportStatusResponse + } + + async getContactsFromList({ listId }: ContactsFromListParams) { + const SENDGRID_EXPORT_TIMEOUT = 10000 + const RETRY_LIMIT = 5 + let numOfRetries = 0 + Logger.debug('Creating contacts exports') + const createContactExport = await this.createContactExport(listId) + const jobId = createContactExport.id + Logger.debug(`Created export with id ${jobId}`) + let exportStatusResponse = await this.getContactExportStatus(jobId) + + do { + Logger.debug('Waiting export to be finished') + await new Promise((r) => setTimeout(r, SENDGRID_EXPORT_TIMEOUT)) + exportStatusResponse = await this.getContactExportStatus(jobId) + Logger.debug(`Export finished with status ${exportStatusResponse.status}`) + switch (exportStatusResponse.status) { + case 'failure': + return Promise.reject(exportStatusResponse.message) + case 'ready': + break + default: + } + numOfRetries++ + } while (exportStatusResponse.status === 'pending' && numOfRetries < RETRY_LIMIT) + if (numOfRetries >= RETRY_LIMIT) { + throw new InternalServerErrorException( + `Couldn't export contacts within the limit. Try again later.`, + ) + } + const exportUrl = exportStatusResponse.urls[0] + const response = await fetch(exportUrl) + + const exportFile = await response.arrayBuffer() + const buffer = Buffer.from(exportFile) + + const contactsList = buffer + .toString() + .trim() + .split('\n') + .map((contact: string) => JSON.parse(contact)) + Logger.debug(`Exported contacts: ${contactsList.length}`) + return contactsList + } + async updateContactList(data: SendGridParams['UpdateListParams']) { const request = { url: `/v3/marketing/lists/${data.id}`, @@ -188,4 +277,53 @@ export class SendGridNotificationsProvider return response } + + async sendBulkEmail(data: MassMailDto, contactsMap: ContactsMap[], value: string): Promise { + const currentDate = new Date() + contactsMap.forEach((contacts, index) => { + //Schedule batches in a minute difference + currentDate.setMinutes(currentDate.getMinutes() + index) + this.sendEmail(data, contacts, currentDate, value) + }) + } + + async sendEmail( + data: MassMailDto, + contacts: ContactsMap, + date: Date, + value: string, + ): Promise { + const personalizations = this.prepareTemplatePersonalizations(data, contacts, date) + const message: MailDataRequired = { + personalizations, + from: this.config.get('SENDGRID_SENDER_EMAIL', ''), + content: [{ type: 'text/html', value: value }], + templateId: data.templateId.trim(), + } + sgMail + .send(message) + .then(() => Logger.debug(`Email sent`)) + .catch((err) => Logger.error(err)) + } + + prepareTemplatePersonalizations( + data: MassMailDto, + contacts: ContactsMap, + date: Date, + ): PersonalizationData[] { + const personalizations: PersonalizationData[] = [] + const scheduleAt = Math.floor(date.getTime() / 1000) + contacts.forEach((mailList, email) => { + personalizations.push({ + to: { email, name: '' }, + dynamicTemplateData: { + subscribe_link: `${process.env.APP_URL}/notifications/subscribe?hash=${mailList.hash}&email=${email}&consent=true`, + unsubscribe_link: `${process.env.APP_URL}/notifications/unsubscribe?email=${email}`, + subject: data.subject, + }, + sendAt: scheduleAt, + }) + }) + return personalizations + } } diff --git a/apps/api/src/notifications/providers/notifications.sendgrid.types.ts b/apps/api/src/notifications/providers/notifications.sendgrid.types.ts index e178a7c53..d4c05a3fd 100644 --- a/apps/api/src/notifications/providers/notifications.sendgrid.types.ts +++ b/apps/api/src/notifications/providers/notifications.sendgrid.types.ts @@ -15,6 +15,7 @@ export type SendGridParams = { RemoveFromUnsubscribedParams: RemoveFromUnsubscribedParams AddToUnsubscribedParams: AddToUnsubscribedParams SendNotificationParams: SendNotificationParams + GetContactsFromListParam: ContactsFromListParams // Responses CreateListRes: string @@ -26,12 +27,16 @@ export type SendGridParams = { RemoveFromUnsubscribedRes: unknown AddToUnsubscribedRes: unknown SendNotificationRes: unknown + GetContactsFromListRes: ContactsResponse[] contactListsRes: SGClientResponse // Implementation specific ContactData: ContactData } +export type ContactsFromListParams = { + listId: string +} type ContactData = { email: string first_name?: string @@ -87,6 +92,46 @@ type GetContactsInfoRes = { [key: string]: { contact: { id: string; [key: string]: unknown; list_ids: string[] } } } +export interface SendGridExportMetadata { + prev: string + self: string + next: string + count: number +} + +export type SendGridExportResponse = { + id: string + _metadata: SendGridExportMetadata +} + +export type SendGridExportStatusResponse = { + id: string + status: 'pending' | 'failure' | 'ready' + created_at: string + updated_at: string + completed_at: string + expires_at: string + urls: string[] + message?: string + _metadata: SendGridExportMetadata + contact_count: number +} + +export interface SendgridExportParams { + list_ids: string[] + file_type: 'json' | 'csv' + segments?: string[] + max_file_size?: number +} + +export interface ContactsResponse { + contact_id: string + created_at: string + custom_fields: object + email: string + updated_at: string +} + type SendGridContactList = { id: string name: string