Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[PRDP-161] feat: Improve temp file handling #37

Merged
merged 9 commits into from
Oct 16, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,20 @@
import it.gov.pagopa.receipt.pdf.generator.service.GenerateReceiptPdfService;
import it.gov.pagopa.receipt.pdf.generator.service.impl.GenerateReceiptPdfServiceImpl;
import it.gov.pagopa.receipt.pdf.generator.utils.ObjectMapperUtils;
import org.apache.commons.io.FileUtils;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;

/**
* Azure Functions with Azure Queue trigger.
Expand All @@ -33,6 +42,9 @@ public class GenerateReceiptPdf {
private final Logger logger = LoggerFactory.getLogger(GenerateReceiptPdf.class);

private static final int MAX_NUMBER_RETRY = Integer.parseInt(System.getenv().getOrDefault("COSMOS_RECEIPT_QUEUE_MAX_RETRY", "5"));
private static final String WORKING_DIRECTORY_PATH = System.getenv().getOrDefault("WORKING_DIRECTORY_PATH", "");

private static final String PATTERN_FORMAT = "yyyy.MM.dd.HH.mm.ss";

private final GenerateReceiptPdfService generateReceiptPdfService;
private final ReceiptCosmosClient receiptCosmosClient;
Expand Down Expand Up @@ -101,7 +113,7 @@ public void processGenerateReceipt(
queueName = "%RECEIPT_QUEUE_TOPIC%",
connection = "RECEIPTS_STORAGE_CONN_STRING")
OutputBinding<String> requeueMessage,
final ExecutionContext context) throws BizEventNotValidException, ReceiptNotFoundException {
final ExecutionContext context) throws BizEventNotValidException, ReceiptNotFoundException, IOException {

//Map queue bizEventMessage to BizEvent
BizEvent bizEvent = getBizEventFromMessage(context, bizEventMessage);
Expand Down Expand Up @@ -142,7 +154,13 @@ public void processGenerateReceipt(
receipt.getId(),
bizEvent.getId());
//Generate and save PDF
PdfGeneration pdfGeneration = generateReceiptPdfService.generateReceipts(receipt, bizEvent);
PdfGeneration pdfGeneration;
Path workingDirPath = createWorkingDirectory();
try {
pdfGeneration = generateReceiptPdfService.generateReceipts(receipt, bizEvent, workingDirPath);
} finally {
deleteTempFolder(workingDirPath);
}

//Verify PDF generation success
boolean success = generateReceiptPdfService.verifyAndUpdateReceipt(receipt, pdfGeneration);
Expand Down Expand Up @@ -205,4 +223,25 @@ private BizEvent getBizEventFromMessage(ExecutionContext context, String bizEven
throw new BizEventNotValidException(errorMsg, e);
}
}

private Path createWorkingDirectory() throws IOException {
File workingDirectory = new File(WORKING_DIRECTORY_PATH);
if (!workingDirectory.exists()) {
try {
Files.createDirectory(workingDirectory.toPath());
} catch (FileAlreadyExistsException ignored) {}
}
return Files.createTempDirectory(workingDirectory.toPath(),
DateTimeFormatter.ofPattern(PATTERN_FORMAT)
.withZone(ZoneId.systemDefault())
.format(Instant.now()));
}

private void deleteTempFolder(Path workingDirPath) {
try {
FileUtils.deleteDirectory(workingDirPath.toFile());
} catch (IOException e) {
logger.warn("Unable to clear working directory: {}", workingDirPath, e);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
import it.gov.pagopa.receipt.pdf.generator.model.request.PdfEngineRequest;
import it.gov.pagopa.receipt.pdf.generator.model.response.PdfEngineResponse;

import java.nio.file.Path;

public interface PdfEngineClient {

PdfEngineResponse generatePDF(PdfEngineRequest pdfEngineRequest);
PdfEngineResponse generatePDF(PdfEngineRequest pdfEngineRequest, Path workingDirPath);
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;

/**
* Client for the PDF Engine
Expand All @@ -33,7 +33,6 @@ public class PdfEngineClientImpl implements PdfEngineClient {

private final String pdfEngineEndpoint = System.getenv().getOrDefault("PDF_ENGINE_ENDPOINT", "");
private final String ocpAimSubKey = System.getenv().getOrDefault("OCP_APIM_SUBSCRIPTION_KEY", "");
private final String workingDirectoryPath = System.getenv().getOrDefault("WORKING_DIRECTORY_PATH", "");

private static final String HEADER_AUTH_KEY = "Ocp-Apim-Subscription-Key";
private static final String ZIP_FILE_NAME = "template.zip";
Expand Down Expand Up @@ -64,7 +63,8 @@ public static PdfEngineClientImpl getInstance() {
* @param pdfEngineRequest Request to the client
* @return response with the PDF or error message and the status
*/
public PdfEngineResponse generatePDF(PdfEngineRequest pdfEngineRequest) {
@Override
public PdfEngineResponse generatePDF(PdfEngineRequest pdfEngineRequest, Path workingDirPath) {

PdfEngineResponse pdfEngineResponse = new PdfEngineResponse();

Expand All @@ -86,7 +86,7 @@ public PdfEngineResponse generatePDF(PdfEngineRequest pdfEngineRequest) {
request.setHeader(HEADER_AUTH_KEY, ocpAimSubKey);
request.setEntity(entity);

pdfEngineResponse = handlePdfEngineResponse(client, request);
pdfEngineResponse = handlePdfEngineResponse(client, request, workingDirPath);
} catch (IOException e) {
handleExceptionErrorMessage(pdfEngineResponse, e);
}
Expand All @@ -101,7 +101,7 @@ public PdfEngineResponse generatePDF(PdfEngineRequest pdfEngineRequest) {
* @param request The request to the PDF engine
* @return pdf engine response
*/
private PdfEngineResponse handlePdfEngineResponse(CloseableHttpClient client, HttpPost request) {
private PdfEngineResponse handlePdfEngineResponse(CloseableHttpClient client, HttpPost request, Path workingDirPath) {
PdfEngineResponse pdfEngineResponse = new PdfEngineResponse();
//Execute call
try (CloseableHttpResponse response = client.execute(request)) {
Expand All @@ -113,7 +113,7 @@ private PdfEngineResponse handlePdfEngineResponse(CloseableHttpClient client, Ht
try (InputStream inputStream = entityResponse.getContent()) {
pdfEngineResponse.setStatusCode(HttpStatus.SC_OK);

saveTempPdf(pdfEngineResponse, inputStream);
saveTempPdf(pdfEngineResponse, inputStream, workingDirPath);
}
} else {
pdfEngineResponse.setStatusCode(HttpStatus.SC_INTERNAL_SERVER_ERROR);
Expand All @@ -134,18 +134,12 @@ private PdfEngineResponse handlePdfEngineResponse(CloseableHttpClient client, Ht
* @param inputStream InputStream pdf
* @throws IOException In case of error to save
*/
private void saveTempPdf(PdfEngineResponse pdfEngineResponse, InputStream inputStream) throws IOException {
File tempDirectory = new File(workingDirectoryPath);
if (!tempDirectory.exists()) {
Files.createDirectory(tempDirectory.toPath());
}

File targetFile = File.createTempFile("tempFile", ".pdf", tempDirectory);
private void saveTempPdf(PdfEngineResponse pdfEngineResponse, InputStream inputStream, Path workingDirPath) throws IOException {
File targetFile = File.createTempFile("tempFile", ".pdf", workingDirPath.toFile());

FileUtils.copyInputStreamToFile(inputStream, targetFile);

pdfEngineResponse.setTempPdfPath(targetFile.getAbsolutePath());
pdfEngineResponse.setTempDirectoryPath(tempDirectory.getAbsolutePath());
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
@NoArgsConstructor
public class PdfEngineResponse {

String tempDirectoryPath;
String tempPdfPath;
int statusCode;
String errorMessage;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import it.gov.pagopa.receipt.pdf.generator.entity.receipt.Receipt;
import it.gov.pagopa.receipt.pdf.generator.model.PdfGeneration;

import java.nio.file.Path;

public interface GenerateReceiptPdfService {

/**
Expand All @@ -13,7 +15,7 @@ public interface GenerateReceiptPdfService {
* @param bizEvent Biz-event from queue message
* @return {@link PdfGeneration} object with the result of the PDF generation and store or the relatives error messages
*/
PdfGeneration generateReceipts(Receipt receipt, BizEvent bizEvent);
PdfGeneration generateReceipts(Receipt receipt, BizEvent bizEvent, Path workingDirPath);

/**
* Verifies if the PDF generation process succeeded or not, and update the receipt with the result
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
Expand Down Expand Up @@ -63,7 +64,7 @@ public GenerateReceiptPdfServiceImpl(PdfEngineClient pdfEngineClient, ReceiptBlo
* {@inheritDoc}
*/
@Override
public PdfGeneration generateReceipts(Receipt receipt, BizEvent bizEvent) {
public PdfGeneration generateReceipts(Receipt receipt, BizEvent bizEvent, Path workingDirPath) {
PdfGeneration pdfGeneration = new PdfGeneration();

String debtorCF = receipt.getEventData().getDebtorFiscalCode();
Expand All @@ -79,7 +80,7 @@ public PdfGeneration generateReceipts(Receipt receipt, BizEvent bizEvent) {
return pdfGeneration;
}
ReceiptPDFTemplate completeTemplate = buildTemplate(bizEvent, false);
PdfMetadata generationResult = generateAndSavePDFReceipt(bizEvent, PAYER_TEMPLATE_SUFFIX, completeTemplate);
PdfMetadata generationResult = generateAndSavePDFReceipt(bizEvent, PAYER_TEMPLATE_SUFFIX, completeTemplate, workingDirPath);
pdfGeneration.setDebtorMetadata(generationResult);
return pdfGeneration;
}
Expand All @@ -90,7 +91,7 @@ public PdfGeneration generateReceipts(Receipt receipt, BizEvent bizEvent) {
} else {
ReceiptPDFTemplate completeTemplate = buildTemplate(bizEvent, false);

PdfMetadata generationResult = generateAndSavePDFReceipt(bizEvent, PAYER_TEMPLATE_SUFFIX, completeTemplate);
PdfMetadata generationResult = generateAndSavePDFReceipt(bizEvent, PAYER_TEMPLATE_SUFFIX, completeTemplate, workingDirPath);
pdfGeneration.setPayerMetadata(generationResult);
}
} else {
Expand All @@ -103,7 +104,7 @@ public PdfGeneration generateReceipts(Receipt receipt, BizEvent bizEvent) {
} else {
ReceiptPDFTemplate onlyDebtorTemplate = buildTemplate(bizEvent, true);

PdfMetadata generationResult = generateAndSavePDFReceipt(bizEvent, DEBTOR_TEMPLATE_SUFFIX, onlyDebtorTemplate);
PdfMetadata generationResult = generateAndSavePDFReceipt(bizEvent, DEBTOR_TEMPLATE_SUFFIX, onlyDebtorTemplate, workingDirPath);
pdfGeneration.setDebtorMetadata(generationResult);
}

Expand Down Expand Up @@ -158,11 +159,11 @@ public boolean verifyAndUpdateReceipt(Receipt receipt, PdfGeneration pdfGenerati
return result;
}

private PdfMetadata generateAndSavePDFReceipt(BizEvent bizEvent, String templateSuffix, ReceiptPDFTemplate completeTemplate) {
private PdfMetadata generateAndSavePDFReceipt(BizEvent bizEvent, String templateSuffix, ReceiptPDFTemplate completeTemplate, Path workingDirPath) {
try {
String dateFormatted = LocalDate.now().format(DateTimeFormatter.ofPattern("yyMMdd"));
String blobName = String.format("%s-%s-%s-%s", TEMPLATE_PREFIX, dateFormatted, bizEvent.getId(), templateSuffix);
PdfEngineResponse pdfEngineResponse = generatePdf(completeTemplate);
PdfEngineResponse pdfEngineResponse = generatePDFReceipt(completeTemplate, workingDirPath);
return saveToBlobStorage(pdfEngineResponse, blobName);
} catch (PDFReceiptGenerationException e) {
logger.error("An error occurred when generating or saving the PDF receipt for biz-event {}", bizEvent.getId(), e);
Expand All @@ -172,16 +173,13 @@ private PdfMetadata generateAndSavePDFReceipt(BizEvent bizEvent, String template

private PdfMetadata saveToBlobStorage(PdfEngineResponse pdfEngineResponse, String blobName) throws SavePDFToBlobException {
String tempPdfPath = pdfEngineResponse.getTempPdfPath();
String tempDirectoryPath = pdfEngineResponse.getTempDirectoryPath();

BlobStorageResponse blobStorageResponse;
//Save to Blob Storage
try (BufferedInputStream pdfStream = new BufferedInputStream(new FileInputStream(tempPdfPath))) {
blobStorageResponse = receiptBlobClient.savePdfToBlobStorage(pdfStream, blobName);
} catch (Exception e) {
throw new SavePDFToBlobException("Error saving pdf to blob storage", ReasonErrorCode.ERROR_BLOB_STORAGE.getCode(), e);
} finally {
deleteTempFolderAndFile(tempPdfPath, tempDirectoryPath);
}

if (blobStorageResponse.getStatusCode() != com.microsoft.azure.functions.HttpStatus.CREATED.value()) {
Expand All @@ -198,8 +196,7 @@ private PdfMetadata saveToBlobStorage(PdfEngineResponse pdfEngineResponse, Strin
.build();
}

// TODO fix temp files create a temp dir and pass it to the client
private PdfEngineResponse generatePdf(ReceiptPDFTemplate template) throws PDFReceiptGenerationException {
private PdfEngineResponse generatePDFReceipt(ReceiptPDFTemplate template, Path workingDirPath) throws PDFReceiptGenerationException {
PdfEngineRequest request = new PdfEngineRequest();

URL templateStream = GenerateReceiptPdfServiceImpl.class.getClassLoader().getResource("template.zip");
Expand All @@ -208,7 +205,7 @@ private PdfEngineResponse generatePdf(ReceiptPDFTemplate template) throws PDFRec
request.setData(parseTemplateDataToString(template));
request.setApplySignature(false);

PdfEngineResponse pdfEngineResponse = pdfEngineClient.generatePDF(request);
PdfEngineResponse pdfEngineResponse = pdfEngineClient.generatePDF(request, workingDirPath);

if (pdfEngineResponse.getStatusCode() != HttpStatus.SC_OK) {
throw new GeneratePDFException(pdfEngineResponse.getErrorMessage(), pdfEngineResponse.getStatusCode());
Expand Down Expand Up @@ -272,26 +269,6 @@ private ReceiptPDFTemplate buildTemplate(BizEvent bizEvent, boolean partialTempl
.build();
}

private void deleteTempFolderAndFile(String tempPdfPath, String tempDirectoryPath) {
File tempFile = new File(tempPdfPath);
if (tempFile.exists()) {
try {
Files.delete(tempFile.toPath());
} catch (IOException e) {
logger.warn("Error deleting temporary pdf file from file system", e);
}
}

File tempDirectory = new File(tempDirectoryPath);
if (tempDirectory.exists()) {
try {
Files.delete(tempDirectory.toPath());
} catch (IOException e) {
logger.warn("Error deleting temporary pdf directory from file system", e);
}
}
}

private String parseTemplateDataToString(ReceiptPDFTemplate template) throws GeneratePDFException {
try {
return ObjectMapperUtils.writeValueAsString(template);
Expand Down
Loading
Loading