Merge pull request #68 from cisagov/SQS-fixes

Clean up P&E's SQS process
cisagov · Mar 27, 2024 · 7323ace · 7323ace
2 parents ddcf403 + 83bbe6c
commit 7323ace
Show file tree

Hide file tree

Showing 17 changed files with 148 additions and 1,253 deletions.
diff --git a/backend/env.yml b/backend/env.yml
@@ -50,17 +50,13 @@ staging:
   REPORTS_BUCKET_NAME: cisa-crossfeed-staging-reports
   CLOUDWATCH_BUCKET_NAME: cisa-crossfeed-staging-cloudwatch
   STAGE: staging
-  PE_CLUSTER_NAME: pe-staging-worker
+  PE_FARGATE_CLUSTER_NAME: pe-staging-worker
+  PE_FARGATE_TASK_DEFINITION_NAME: pe-staging-worker
   SHODAN_QUEUE_URL: ${ssm:/crossfeed/staging/SHODAN_QUEUE_URL}
-  SHODAN_SERVICE_NAME: pe-staging-shodan
   DNSTWIST_QUEUE_URL: ${ssm:/crossfeed/staging/DNSTWIST_QUEUE_URL}
-  DNSTWIST_SERVICE_NAME: pe-staging-dnstwist
   HIBP_QUEUE_URL: ${ssm:/crossfeed/staging/HIBP_QUEUE_URL}
-  HIBP_SERVICE_NAME: pe-staging-hibp
   INTELX_QUEUE_URL: ${ssm:/crossfeed/staging/INTELX_QUEUE_URL}
-  INTELX_SERVICE_NAME: pe-staging-intelx
   CYBERSIXGILL_QUEUE_URL: ${ssm:/crossfeed/staging/CYBERSIXGILL_QUEUE_URL}
-  CYBERSIXGILL_SERVICE_NAME: pe-staging-cybersixgill
   EMAIL_BUCKET_NAME: cisa-crossfeed-staging-html-email
 
 prod:
@@ -103,9 +99,13 @@ prod:
   REPORTS_BUCKET_NAME: cisa-crossfeed-prod-reports
   CLOUDWATCH_BUCKET_NAME: cisa-crossfeed-prod-cloudwatch
   STAGE: prod
-  PE_CLUSTER_NAME: pe-prod-worker
+  PE_FARGATE_CLUSTER_NAME: pe-prod-worker
+  PE_FARGATE_TASK_DEFINITION_NAME: pe-prod-worker
   SHODAN_QUEUE_URL: ${ssm:/crossfeed/prod/SHODAN_QUEUE_URL}
-  SHODAN_SERVICE_NAME: pe-prod-shodan
+  DNSTWIST_QUEUE_URL: ${ssm:/crossfeed/prod/DNSTWIST_QUEUE_URL}
+  HIBP_QUEUE_URL: ${ssm:/crossfeed/prod/HIBP_QUEUE_URL}
+  INTELX_QUEUE_URL: ${ssm:/crossfeed/prod/INTELX_QUEUE_URL}
+  CYBERSIXGILL_QUEUE_URL: ${ssm:/crossfeed/prod/CYBERSIXGILL_QUEUE_URL}
   EMAIL_BUCKET_NAME: cisa-crossfeed-staging-html-email
 
 dev-vpc:

diff --git a/backend/package.json b/backend/package.json
@@ -111,6 +111,8 @@
     "lint": "eslint '**/*.{ts,tsx,js,jsx}'",
     "lint:fix": "eslint '**/*.{ts,tsx,js,jsx}' --fix",
     "pesyncdb": "docker-compose exec -T backend npx ts-node src/tools/run-pesyncdb.ts",
+    "scan-exec": "docker-compose exec -T backend npx ts-node src/tools/run-scanExecution.ts",
+    "send-message": "node sendMessage.js",
     "syncdb": "docker-compose exec -T backend npx ts-node src/tools/run-syncdb.ts",
     "syncmdl": "docker-compose exec -T backend npx ts-node src/tools/run-syncmdl.ts",
     "test": "jest --detectOpenHandles",

diff --git a/backend/sendMessage.js b/backend/sendMessage.js
@@ -1,19 +1,18 @@
 // sendMessage.js
 const amqp = require('amqplib');
 
-async function sendMessageToControlQueue(message) {
+async function sendMessageToQueue(message, queue) {
   const connection = await amqp.connect('amqp://localhost');
   const channel = await connection.createChannel();
-  const controlQueue = 'ControlQueue';
 
-  await channel.assertQueue(controlQueue, { durable: true });
+  await channel.assertQueue(queue, { durable: true });
 
-  // Simulate sending a message to the ControlQueue
-  channel.sendToQueue(controlQueue, Buffer.from(JSON.stringify(message)), {
+  // Simulate sending a message to the queue
+  channel.sendToQueue(queue, Buffer.from(JSON.stringify(message)), {
     persistent: true
   });
 
-  console.log('Message sent to ControlQueue:', message);
+  console.log('Message sent:', message);
 
   setTimeout(() => {
     connection.close();
@@ -22,7 +21,12 @@ async function sendMessageToControlQueue(message) {
 
 // Simulate sending a message
 const message = {
-  scriptType: 'shodan',
+  scriptType: 'dnstwist',
   org: 'DHS'
 };
-sendMessageToControlQueue(message);
+const queue = 'dnstwistQueue';
+sendMessageToQueue(message, queue);
+sendMessageToQueue(message, queue);
+sendMessageToQueue(message, queue);
+sendMessageToQueue(message, queue);
+sendMessageToQueue(message, queue);
diff --git a/backend/serverless.yml b/backend/serverless.yml
@@ -108,13 +108,6 @@ provider:
 
 resources:
   Resources:
-    WorkerControlQueue:
-      Type: AWS::SQS::Queue
-      Properties:
-        QueueName: ${self:provider.stage}-worker-control-queue
-        VisibilityTimeout: 300  # Should match or exceed function timeout
-        MaximumMessageSize: 262144  # 256 KB
-        MessageRetentionPeriod: 604800  # 7 days
     ShodanQueue:
       Type: AWS::SQS::Queue
       Properties:

diff --git a/backend/src/tasks/ecs-client.ts b/backend/src/tasks/ecs-client.ts
@@ -81,7 +81,7 @@ class ECSClient {
             // In order to use the host name "db" to access the database from the
             // crossfeed-worker image, we must launch the Docker container with
             // the Crossfeed backend network.
-            NetworkMode: 'crossfeed_backend',
+            NetworkMode: 'xfd_backend',
             Memory: 4000000000 // Limit memory to 4 GB. We do this locally to better emulate fargate memory conditions. TODO: In the future, we could read the exact memory from SCAN_SCHEMA to better emulate memory requirements for each scan.
           },
           Env: [

diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml
@@ -39,17 +39,10 @@ checkUserExpiration:
   handler: src/tasks/checkUserExpiration.handler
   events:
     - schedule: cron(0 0 * * ? *)  # Runs every day at midnight
+
 scanExecution:
+  timeout: 900  # 15 minutes
   handler: src/tasks/scanExecution.handler
-  timeout: 300  # 5 minutes
-  environment:
-    SQS_QUEUE_NAME: ${self:provider.stage}-worker-control-queue
-  events:
-    - sqs:
-        arn:
-          Fn::GetAtt:
-            - WorkerControlQueue
-            - Arn
   memorySize: 4096
 
 updateScanTaskStatus: