Fix QueueGranules timeouts (#318)

Fixes #317
NASA-IMPACT · Jan 3, 2024 · 055916a · 055916a
1 parent 54bc70e
commit 055916a
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 5 deletions.
diff --git a/_templates/rule/new/rule.ejs.t b/_templates/rule/new/rule.ejs.t
@@ -28,6 +28,7 @@ message: >
     "ingestedPathFormat": "<%- ingestedPathFormat %>",
     "startDate": "<%= startYear %>-01-01T00:00:00Z",
     "endDate": "<%= parseInt(endYear) + 1 %>-01-01T00:00:00Z",
-    "step": "P1D"
+    "step": "P1D",
+    "maxBatchSize": 200,
   }
 }
diff --git a/app/stacks/cumulus/main.tf b/app/stacks/cumulus/main.tf
@@ -161,7 +161,7 @@ resource "aws_cloudwatch_event_target" "background_job_queue_watcher" {
   arn  = module.cumulus.sqs2sfThrottle_lambda_function_arn
   input = jsonencode(
     {
-      messageLimit = 500
+      messageLimit = 1500
       queueUrl     = aws_sqs_queue.background_job_queue.id
       timeLimit    = 60
     }
@@ -576,7 +576,7 @@ module "cumulus" {
     {
       id              = "backgroundJobQueue",
       url             = aws_sqs_queue.background_job_queue.id,
-      execution_limit = 300
+      execution_limit = 500
     }
   ]
 }
diff --git a/app/stacks/cumulus/templates/discover-granules-workflow.asl.json b/app/stacks/cumulus/templates/discover-granules-workflow.asl.json
@@ -287,6 +287,7 @@
                                 "TargetPath": "$.payload"
                               },
                               "task_config": {
+                                "concurrency": 1,
                                 "queueUrl": "${background_job_queue_url}",
                                 "preferredQueueBatchSize": "{$.meta.collection.meta.preferredQueueBatchSize}",
                                 "provider": "{$.meta.provider}",

diff --git a/src/lib/discovery.ts b/src/lib/discovery.ts
@@ -61,13 +61,13 @@ type BucketKey = {
   readonly key: string;
 };
 
-// Default maximum batch size for batching granules after discovery is 1000, but this
+// Default maximum batch size for batching granules after discovery is 200, but this
 // can be set on a per rule basis by setting `meta.maxBatchSize` in a rule definition.
 export const BatchGranulesInput = t.readonly(
   t.type({
     config: t.type({
       providerPath: t.string,
-      maxBatchSize: tt.fromNullable(t.number, 1000),
+      maxBatchSize: tt.fromNullable(t.number, 200),
     }),
     input: DiscoverGranulesOutput,
   })