From 6f5fc7b68d0a61cd93d3654083f7c2135bb4d04a Mon Sep 17 00:00:00 2001 From: jsrikish <54687119+jsrikish@users.noreply.github.com> Date: Tue, 30 Jan 2024 11:03:09 -0600 Subject: [PATCH] discover queue concurrency, ingest publish freq, new rule #324 --- app/stacks/cumulus/main.tf | 2 +- .../WV01_Pan_L1B/v1/WV01_Pan_L1B___1.json | 21 +++++++++++++++++++ .../discover-granules-workflow.asl.json | 2 +- to_delete.txt | 1 + 4 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 app/stacks/cumulus/resources/rules/WV01_Pan_L1B/v1/WV01_Pan_L1B___1.json create mode 100644 to_delete.txt diff --git a/app/stacks/cumulus/main.tf b/app/stacks/cumulus/main.tf index 1b22ffa..da44c4b 100644 --- a/app/stacks/cumulus/main.tf +++ b/app/stacks/cumulus/main.tf @@ -576,7 +576,7 @@ module "cumulus" { { id = "backgroundJobQueue", url = aws_sqs_queue.background_job_queue.id, - execution_limit = 1000 + execution_limit = 1200 } ] } diff --git a/app/stacks/cumulus/resources/rules/WV01_Pan_L1B/v1/WV01_Pan_L1B___1.json b/app/stacks/cumulus/resources/rules/WV01_Pan_L1B/v1/WV01_Pan_L1B___1.json new file mode 100644 index 0000000..b737994 --- /dev/null +++ b/app/stacks/cumulus/resources/rules/WV01_Pan_L1B/v1/WV01_Pan_L1B___1.json @@ -0,0 +1,21 @@ +{ + "name": "WV01_Pan_L1B___1", + "state": "DISABLED", + "rule": { + "type": "onetime" + }, + "provider": "cumulus", + "collection": { + "name": "WV01_Pan_L1B", + "version": "1" + }, + "workflow": "DiscoverAndQueueGranules", + "meta": { + "discoverOnly": false, + "providerPathFormat": "'WV01_Pan_L1B___1/'yyyy/DDD", + "ingestedPathFormat": "'WV01_Pan_L1B___1/'yyyy/DDD", + "startDate": "2007-01-01T00:00:00Z", + "endDate": "2023-01-01T00:00:00Z", + "step": "P1D" + } +} diff --git a/app/stacks/cumulus/templates/discover-granules-workflow.asl.json b/app/stacks/cumulus/templates/discover-granules-workflow.asl.json index 84cbaba..4d24b24 100644 --- a/app/stacks/cumulus/templates/discover-granules-workflow.asl.json +++ b/app/stacks/cumulus/templates/discover-granules-workflow.asl.json @@ -30,7 +30,7 @@ "DiscoverGranulesMap": { "Type": "Map", "End": true, - "MaxConcurrency": 3, + "MaxConcurrency": 2, "ToleratedFailurePercentage": 1, "ItemReader": { "Resource": "arn:aws:states:::s3:getObject", diff --git a/to_delete.txt b/to_delete.txt new file mode 100644 index 0000000..972f24a --- /dev/null +++ b/to_delete.txt @@ -0,0 +1 @@ +delete this file