Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

EC2 Instance sync => new data model #1146

Merged
merged 3 commits into from
Mar 31, 2023
+1,066 −576
Merged

EC2 Instance sync => new data model #1146

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e52deb3
EC2 Instance sync => new data model
achantavy Mar 20, 2023
940149c
Remove incorrect comment
achantavy Mar 20, 2023
9487d38
Use namedtuples, delete dead code
achantavy Mar 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 0 additions & 18 deletions cartography/data/indexes.cypher
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,27 +97,11 @@ CREATE INDEX IF NOT EXISTS FOR (n:DynamoDBTable) ON (n.id);
CREATE INDEX IF NOT EXISTS FOR (n:DynamoDBTable) ON (n.lastupdated);
CREATE INDEX IF NOT EXISTS FOR (n:EBSSnapshot) ON (n.id);
CREATE INDEX IF NOT EXISTS FOR (n:EBSSnapshot) ON (n.lastupdated);
CREATE INDEX IF NOT EXISTS FOR (n:EBSVolume) ON (n.id);
CREATE INDEX IF NOT EXISTS FOR (n:EBSVolume) ON (n.lastupdated);
CREATE INDEX IF NOT EXISTS FOR (n:EC2Instance) ON (n.id);
CREATE INDEX IF NOT EXISTS FOR (n:EC2Instance) ON (n.instanceid);
CREATE INDEX IF NOT EXISTS FOR (n:EC2Instance) ON (n.publicdnsname);
CREATE INDEX IF NOT EXISTS FOR (n:EC2Instance) ON (n.lastupdated);
CREATE INDEX IF NOT EXISTS FOR (n:EC2KeyPair) ON (n.id);
CREATE INDEX IF NOT EXISTS FOR (n:EC2KeyPair) ON (n.keyfingerprint);
CREATE INDEX IF NOT EXISTS FOR (n:EC2KeyPair) ON (n.lastupdated);
CREATE INDEX IF NOT EXISTS FOR (n:EC2PrivateIp) ON (n.id);
CREATE INDEX IF NOT EXISTS FOR (n:EC2PrivateIp) ON (n.lastupdated);
CREATE INDEX IF NOT EXISTS FOR (n:EC2Reservation) ON (n.reservationid);
CREATE INDEX IF NOT EXISTS FOR (n:EC2Reservation) ON (n.lastupdated);
CREATE INDEX IF NOT EXISTS FOR (n:EC2ReservedInstance) ON (n.id);
CREATE INDEX IF NOT EXISTS FOR (n:EC2ReservedInstance) ON (n.lastupdated);
CREATE INDEX IF NOT EXISTS FOR (n:EC2SecurityGroup) ON (n.groupid);
CREATE INDEX IF NOT EXISTS FOR (n:EC2SecurityGroup) ON (n.id);
CREATE INDEX IF NOT EXISTS FOR (n:EC2SecurityGroup) ON (n.lastupdated);
CREATE INDEX IF NOT EXISTS FOR (n:EC2Subnet) ON (n.id);
CREATE INDEX IF NOT EXISTS FOR (n:EC2Subnet) ON (n.subnetid);
CREATE INDEX IF NOT EXISTS FOR (n:EC2Subnet) ON (n.lastupdated);
CREATE INDEX IF NOT EXISTS FOR (n:ECRImage) ON (n.id);
CREATE INDEX IF NOT EXISTS FOR (n:ECRImage) ON (n.digest);
CREATE INDEX IF NOT EXISTS FOR (n:ECRImage) ON (n.lastupdated);
Expand Down Expand Up @@ -241,8 +225,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:LoadBalancer) ON (n.lastupdated);
CREATE INDEX IF NOT EXISTS FOR (n:LoadBalancerV2) ON (n.dnsname);
CREATE INDEX IF NOT EXISTS FOR (n:LoadBalancerV2) ON (n.id);
CREATE INDEX IF NOT EXISTS FOR (n:LoadBalancerV2) ON (n.lastupdated);
CREATE INDEX IF NOT EXISTS FOR (n:NetworkInterface) ON (n.id);
CREATE INDEX IF NOT EXISTS FOR (n:NetworkInterface) ON (n.lastupdated);
CREATE INDEX IF NOT EXISTS FOR (n:NameServer) ON (n.id);
CREATE INDEX IF NOT EXISTS FOR (n:NameServer) ON (n.lastupdated);
CREATE INDEX IF NOT EXISTS FOR (n:OktaOrganization) ON (n.id);
Expand Down
68 changes: 0 additions & 68 deletions cartography/data/jobs/cleanup/aws_import_ec2_instances_cleanup.json
View file
Open in desktop

This file was deleted.

7 changes: 5 additions & 2 deletions cartography/graph/cleanupbuilder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
from dataclasses import asdict
from string import Template
from typing import Dict
from typing import List

from cartography.graph.querybuilder import _build_match_clause
from cartography.graph.querybuilder import rel_present_on_node_schema
from cartography.models.core.common import PropertyRef
from cartography.models.core.nodes import CartographyNodeSchema
from cartography.models.core.relationships import CartographyRelSchema
from cartography.models.core.relationships import LinkDirection
Expand Down Expand Up @@ -150,11 +152,12 @@ def _validate_target_node_matcher_for_cleanup_job(tgm: TargetNodeMatcher):
class injects the sub resource id via a query kwarg parameter. See GraphJob and GraphStatement classes.
This is a private function meant only to be called when we clean up the sub resource relationship.
"""
tgm_asdict = asdict(tgm)
tgm_asdict: Dict[str, PropertyRef] = asdict(tgm)

for key, prop_ref in tgm_asdict.items():
if not prop_ref.set_in_kwargs:
raise ValueError(
f"TargetNodeMatcher PropertyRefs in the sub_resource_relationship must have set_in_kwargs=True. "
f"{key} has set_in_kwargs=False, please check.",
f"{key} has set_in_kwargs=False, please check by reviewing the full stack trace to know which object"
f"this message was raised from. Debug information: PropertyRef name = {prop_ref.name}.",
)
Loading