Dj hot fix 2 (#665)

* filter out domains that don't link to cidrs

filter out domains that don't link to cidrs

* update tests

update tests

* run linter

run linter

* update domain.test.ts

update domain.test.ts with isFceb so tests run as normal

* update all domains.create statements in tests

update all domains.create statements in tests

* fix vuln tests

fix vuln tests

* Update test snapshots

* Update backend andWhere domain filter query with parenthesis

---------

Co-authored-by: aloftus23 <[email protected]>
Co-authored-by: Amelia Vance <[email protected]>

backend/src/api/domains.ts

@@ -154,6 +154,10 @@ class DomainSearch {
       });
     }
 
+    qs.andWhere(
+      '(domain."isFceb" = true OR (domain."isFceb" = false AND domain."fromCidr" = true))'
+    );
+
     await this.filterResultQueryset(qs, event);
     return qs.getManyAndCount();
   }

backend/src/api/scans.ts

@@ -128,6 +128,13 @@ export const SCAN_SCHEMA: ScanSchema = {
     description:
       'Open source tool that integrates passive APIs in order to discover target subdomains'
   },
+  flagFloatingIps: {
+    type: 'fargate',
+    isPassive: true,
+    global: true,
+    description:
+      'Loops through all domains and determines if their associated IP can be found in a report Cidr block.'
+  },
   hibp: {
     type: 'fargate',
     isPassive: true,

backend/src/api/stats.ts

@@ -94,6 +94,10 @@ export const get = wrapHandler(async (event) => {
       });
     }
 
+    qs.andWhere(
+      '(domain."isFceb" = true OR (domain."isFceb" = false AND domain."fromCidr" = true))'
+    );
+
     // Handles the case where no orgs and no regions are set, and we pull stats for a region that will never exist
     if (
       search.filters?.organizations?.length === 0 &&

backend/src/api/vulnerabilities.ts

@@ -173,6 +173,10 @@ class VulnerabilitySearch {
       .leftJoinAndSelect('domain.organization', 'organization')
       .leftJoinAndSelect('vulnerability.service', 'service');
 
+    qs.andWhere(
+      '(domain."isFceb" = true OR (domain."isFceb" = false AND domain."fromCidr" = true))'
+    );
+
     if (groupBy) {
       qs = qs
         .groupBy('title, cve, "isKev", description, severity')

backend/src/models/domain.ts

@@ -111,6 +111,16 @@ export class Domain extends BaseEntity {
   })
   cloudHosted: boolean;
 
+  @Column({
+    default: false
+  })
+  fromCidr: boolean;
+
+  @Column({
+    default: false
+  })
+  isFceb: boolean;
+
   /** SSL Certificate information  */
   @Column({
     type: 'jsonb',

backend/src/tasks/flagFloatingIps.ts

@@ -0,0 +1,28 @@
+import { CommandOptions } from './ecs-client';
+import checkIpInCidr from './helpers/checkIpInCidr';
+import { Organization, connectToDatabase } from '../models';
+
+export const handler = async (commandOptions: CommandOptions) => {
+  const db_connection = await connectToDatabase();
+  const organization_repo = db_connection.getRepository(Organization);
+  const organizations = await organization_repo.find({
+    relations: ['domains']
+  });
+  for (const organization of organizations) {
+    for (const domain of organization.domains) {
+      if (domain.ip) {
+        const cidrSectorDict = await checkIpInCidr(
+          domain.ip,
+          organization.acronym
+        );
+        if (cidrSectorDict['isInCidr']) {
+          domain.fromCidr = true;
+        }
+        if (cidrSectorDict['isExecutive']) {
+          domain.isFceb = true;
+        }
+        domain.save();
+      }
+    }
+  }
+};

backend/src/tasks/helpers/checkIpInCidr.ts

@@ -0,0 +1,61 @@
+import { getRepository } from 'typeorm';
+import { Cidr, DL_Organization, connectToDatalake2 } from '../../models';
+
+export default async (
+  ip: string,
+  acronym: string
+): Promise<{ isInCidr: boolean; isExecutive: boolean }> => {
+  // await connectToDatalake2()
+  // const cidrRepository = getRepository(Cidr);
+  // const organizationRepository = getRepository(DL_Organization);
+
+  // Find the organization by acronym
+  const mdl_connection = await connectToDatalake2();
+  const mdl_organization_repo = mdl_connection.getRepository(DL_Organization);
+  const organization = await mdl_organization_repo.findOne({
+    where: { acronym },
+    relations: ['cidrs', 'sectors', 'parent']
+  });
+
+  if (!organization) {
+    return { isInCidr: false, isExecutive: false };
+  }
+
+  const isOrganizationExecutive = async (
+    org: DL_Organization
+  ): Promise<boolean> => {
+    if (org.sectors.some((sector) => sector.acronym === 'EXECUTIVE')) {
+      return true;
+    }
+    if (org.parent) {
+      const parentOrg = await mdl_organization_repo.findOne({
+        where: { id: org.parent.id },
+        relations: ['sectors']
+      });
+
+      return parentOrg ? await isOrganizationExecutive(parentOrg) : false;
+    }
+    return false;
+  };
+
+  const isExecutive = await isOrganizationExecutive(organization);
+
+  // Get CIDRs related to the organization
+  const cidrs = organization.cidrs.map((cidr) => cidr.network);
+
+  if (cidrs.length === 0) {
+    return { isInCidr: false, isExecutive }; // No CIDRs associated with the organization
+  }
+
+  // Check if the IP is in any of the CIDRs
+  const mdl_cidr_repo = mdl_connection.getRepository(Cidr);
+  const result = await mdl_cidr_repo
+    .createQueryBuilder('cidr')
+    .where('cidr.network >>= :ip', { ip })
+    .andWhere('cidr.id IN (:...cidrIds)', {
+      cidrIds: organization.cidrs.map((cidr) => cidr.id)
+    })
+    .getCount();
+
+  return { isInCidr: result > 0, isExecutive };
+};

backend/src/tasks/search-sync-domains.ts

@@ -40,6 +40,10 @@ export const handler = async (commandOptions: CommandOptions) => {
     qs.where('organization.id=:org', { org: organizationId });
   }
 
+  qs.andWhere(
+    '(domain."isFceb" = true OR (domain."isFceb" = false AND domain."fromCidr" = true))'
+  );
+
   const domainIds = (await qs.getMany()).map((e) => e.id);
   console.log(`Got ${domainIds.length} domains.`);
   if (domainIds.length) {