From a44453e67fa582151835bd90c13f2cd1d491fdc5 Mon Sep 17 00:00:00 2001 From: Alex Chantavy Date: Tue, 14 Mar 2023 13:19:13 -0700 Subject: [PATCH] #1017, #1032, #1115: Neo4j 5.x experimental support (#1033) - Removes deprecated `exists()` function with `is not null`. This is compatible with both Neo4j 4.x and 5.x. - Replaces `CALL db.indexes()` with `SHOW INDEXES` because it was also replaced per https://neo4j.com/docs/operations-manual/5/reference/procedures/#procedure_db_indexes. - Updates install docs. - Updates CI to test against both Neo4j 4.4 and 5.latest docker containers. See - https://neo4j.com/docs/upgrade-migration-guide/current/version-5/migration/breaking-changes/#_removals. - #1115. --- .github/workflows/test_suite.yml | 55 +++++++++++++++---- .../jobs/analysis/aws_ec2_asset_exposure.json | 10 ++-- .../analysis/aws_ec2_keypair_analysis.json | 4 +- .../jobs/analysis/aws_eks_asset_exposure.json | 2 +- .../jobs/analysis/aws_foreign_accounts.json | 4 +- .../gcp_compute_asset_inet_exposure.json | 8 +-- .../jobs/analysis/gcp_gke_asset_exposure.json | 2 +- .../jobs/analysis/gcp_gke_basic_auth.json | 4 +- .../jobs/cleanup/aws_apigateway_details.json | 2 +- .../data/jobs/cleanup/aws_kms_details.json | 2 +- .../data/jobs/cleanup/aws_s3_details.json | 2 +- cartography/intel/aws/apigateway.py | 2 +- cartography/intel/aws/kms.py | 2 +- cartography/intel/aws/s3.py | 4 +- docs/root/dev/writing-analysis-jobs.md | 2 +- docs/root/install.md | 27 ++++++--- tests/integration/test_basic.py | 2 +- 17 files changed, 90 insertions(+), 44 deletions(-) diff --git a/.github/workflows/test_suite.yml b/.github/workflows/test_suite.yml index b33f3b4347..c90ece705c 100644 --- a/.github/workflows/test_suite.yml +++ b/.github/workflows/test_suite.yml @@ -26,18 +26,50 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - # See https://debian.neo4j.com/ - - name: Neo4j setup + # https://stackoverflow.com/a/64592785 + - name: neo4j 4 instance setup run: | - wget -O - https://debian.neo4j.com/neotechnology.gpg.key | sudo apt-key add - - echo 'deb https://debian.neo4j.com stable 4.4' | sudo tee /etc/apt/sources.list.d/neo4j.list - sudo apt-get update - - name: Install Neo4j - run: sudo apt-get install neo4j - - name: Disable auth for neo4j + docker run --detach \ + --name neo4j-4 \ + --env NEO4J_AUTH=none \ + --publish 7474:7474 \ + --publish 7473:7473 \ + --publish 7687:7687 \ + neo4j:4.4-community + - uses: actions/setup-python@v2 + with: + python-version: "3.8" + # Cache our pip dir for efficiency; see https://medium.com/ai2-blog/python-caching-in-github-actions-e9452698e98d. + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ hashFiles('setup.py') }}-${{ hashFiles('test-requirements.txt') }} + - name: Install cartography + run: | + pip install -e . + pip install -r test-requirements.txt + - name: Wait for neo4j 4 to be ready + timeout-minutes: 1 + run: (docker logs -f neo4j-4 & ) | grep -q Started + - name: make test_unit + run: make test_unit + - name: make test_integration + run: make test_integration + + unit-and-integration-tests-neo4j5: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + # https://stackoverflow.com/a/64592785 + - name: neo4j 5 setup run: | - sudo sed -i 's/#dbms.security.auth_enabled=false/dbms.security.auth_enabled=false/g' /etc/neo4j/neo4j.conf - sudo service neo4j restart + docker run --detach \ + --name neo4j-5 \ + --env NEO4J_AUTH=none \ + --publish 7474:7474 \ + --publish 7473:7473 \ + --publish 7687:7687 \ + neo4j:5 - uses: actions/setup-python@v2 with: python-version: "3.8" @@ -50,6 +82,9 @@ jobs: run: | pip install -e . pip install -r test-requirements.txt + - name: Wait for neo4j 5 to be ready + timeout-minutes: 1 + run: (docker logs -f neo4j-5 & ) | grep -q Started - name: make test_unit run: make test_unit - name: make test_integration diff --git a/cartography/data/jobs/analysis/aws_ec2_asset_exposure.json b/cartography/data/jobs/analysis/aws_ec2_asset_exposure.json index 0ca04092af..e5bc56cf34 100644 --- a/cartography/data/jobs/analysis/aws_ec2_asset_exposure.json +++ b/cartography/data/jobs/analysis/aws_ec2_asset_exposure.json @@ -1,12 +1,12 @@ { "statements": [ { - "query": "MATCH (n) where EXISTS(n.exposed_internet) AND labels(n) IN ['AutoScalingGroup', 'EC2Instance', 'LoadBalancer', 'LoadBalancerV2'] WITH n LIMIT $LIMIT_SIZE REMOVE n.exposed_internet, n.exposed_internet_type return COUNT(*) as TotalCompleted", + "query": "MATCH (n) where n.exposed_internet IS NOT NULL AND labels(n) IN ['AutoScalingGroup', 'EC2Instance', 'LoadBalancer', 'LoadBalancerV2'] WITH n LIMIT $LIMIT_SIZE REMOVE n.exposed_internet, n.exposed_internet_type return COUNT(*) as TotalCompleted", "iterative": true, "iterationsize": 1000 }, { - "query": "MATCH (:IpRange{id: '0.0.0.0/0'})-[:MEMBER_OF_IP_RULE]->(:IpPermissionInbound)-[:MEMBER_OF_EC2_SECURITY_GROUP]->(group:EC2SecurityGroup)<-[:MEMBER_OF_EC2_SECURITY_GROUP|NETWORK_INTERFACE*..2]-(instance:EC2Instance)\nWITH instance\nWHERE (EXISTS(instance.publicipaddress)) AND (NOT EXISTS(instance.exposed_internet_type)) OR (NOT 'direct' IN instance.exposed_internet_type)\nSET instance.exposed_internet = true, instance.exposed_internet_type = coalesce(instance.exposed_internet_type , []) + 'direct';", + "query": "MATCH (:IpRange{id: '0.0.0.0/0'})-[:MEMBER_OF_IP_RULE]->(:IpPermissionInbound)-[:MEMBER_OF_EC2_SECURITY_GROUP]->(group:EC2SecurityGroup)<-[:MEMBER_OF_EC2_SECURITY_GROUP|NETWORK_INTERFACE*..2]-(instance:EC2Instance)\nWITH instance\nWHERE (instance.publicipaddress IS NOT NULL) AND (instance.exposed_internet_type IS NULL) OR (NOT 'direct' IN instance.exposed_internet_type)\nSET instance.exposed_internet = true, instance.exposed_internet_type = coalesce(instance.exposed_internet_type , []) + 'direct';", "iterative": false }, { @@ -18,15 +18,15 @@ "iterative": false }, { - "query": "MATCH (elb:LoadBalancer{exposed_internet: true})-[:EXPOSE]->(e:EC2Instance)\nWITH e\nWHERE (NOT EXISTS(e.exposed_internet_type)) OR (NOT 'elb' IN e.exposed_internet_type)\nSET e.exposed_internet = true, e.exposed_internet_type = coalesce(e.exposed_internet_type, []) + 'elb'", + "query": "MATCH (elb:LoadBalancer{exposed_internet: true})-[:EXPOSE]->(e:EC2Instance)\nWITH e\nWHERE (e.exposed_internet_type IS NULL) OR (NOT 'elb' IN e.exposed_internet_type)\nSET e.exposed_internet = true, e.exposed_internet_type = coalesce(e.exposed_internet_type, []) + 'elb'", "iterative": false }, { - "query": "MATCH (elbv2:LoadBalancerV2{exposed_internet: true})-[:EXPOSE]->(e:EC2Instance)\nWITH e\nWHERE (NOT EXISTS(e.exposed_internet_type)) OR (NOT 'elbv2' IN e.exposed_internet_type)\nSET e.exposed_internet = true, e.exposed_internet_type = coalesce(e.exposed_internet_type, []) + 'elbv2'", + "query": "MATCH (elbv2:LoadBalancerV2{exposed_internet: true})-[:EXPOSE]->(e:EC2Instance)\nWITH e\nWHERE (e.exposed_internet_type IS NULL) OR (NOT 'elbv2' IN e.exposed_internet_type)\nSET e.exposed_internet = true, e.exposed_internet_type = coalesce(e.exposed_internet_type, []) + 'elbv2'", "iterative": false }, { - "query": "MATCH (instance:EC2Instance{exposed_internet: true})-[:MEMBER_AUTO_SCALE_GROUP]->(asg:AutoScalingGroup)\nWITH distinct instance.exposed_internet_type as types, asg\nUNWIND types as type\nWITH type, asg\nWHERE NOT EXISTS(asg.exposed_internet_type) OR (NOT type IN asg.exposed_internet_type)\nSET asg.exposed_internet = true, asg.exposed_internet_type = coalesce(asg.exposed_internet_type, []) + type;", + "query": "MATCH (instance:EC2Instance{exposed_internet: true})-[:MEMBER_AUTO_SCALE_GROUP]->(asg:AutoScalingGroup)\nWITH distinct instance.exposed_internet_type as types, asg\nUNWIND types as type\nWITH type, asg\nWHERE asg.exposed_internet_type IS NULL OR (NOT type IN asg.exposed_internet_type)\nSET asg.exposed_internet = true, asg.exposed_internet_type = coalesce(asg.exposed_internet_type, []) + type;", "iterative": false } ], diff --git a/cartography/data/jobs/analysis/aws_ec2_keypair_analysis.json b/cartography/data/jobs/analysis/aws_ec2_keypair_analysis.json index dfff01a07b..dc8952ddbb 100644 --- a/cartography/data/jobs/analysis/aws_ec2_keypair_analysis.json +++ b/cartography/data/jobs/analysis/aws_ec2_keypair_analysis.json @@ -3,12 +3,12 @@ "statements": [ { "__comment__": "Delete the attribute user_uploaded", - "query": "MATCH (k:EC2KeyPair) WHERE EXISTS (k.user_uploaded) REMOVE k.user_uploaded return COUNT(*) as TotalCompleted", + "query": "MATCH (k:EC2KeyPair) WHERE k.user_uploaded IS NOT NULL REMOVE k.user_uploaded return COUNT(*) as TotalCompleted", "iterative": false }, { "__comment__": "Delete the attribute duplicate_keyfingerprint", - "query": "MATCH (k:EC2KeyPair) WHERE EXISTS (k.duplicate_keyfingerprint) REMOVE k.duplicate_keyfingerprint return COUNT(*) as TotalCompleted", + "query": "MATCH (k:EC2KeyPair) WHERE k.duplicate_keyfingerprint IS NOT NULL REMOVE k.duplicate_keyfingerprint return COUNT(*) as TotalCompleted", "iterative": false }, { diff --git a/cartography/data/jobs/analysis/aws_eks_asset_exposure.json b/cartography/data/jobs/analysis/aws_eks_asset_exposure.json index fe744f6614..44a7bf8d36 100644 --- a/cartography/data/jobs/analysis/aws_eks_asset_exposure.json +++ b/cartography/data/jobs/analysis/aws_eks_asset_exposure.json @@ -2,7 +2,7 @@ "statements": [ { "__comment": "This is a clean-up statement to remove custom attributes", - "query": "MATCH (cluster:EKSCluster) WHERE EXISTS(cluster.exposed_internet) REMOVE cluster.exposed_internet return COUNT(*) as TotalCompleted", + "query": "MATCH (cluster:EKSCluster) WHERE cluster.exposed_internet IS NOT NULL REMOVE cluster.exposed_internet return COUNT(*) as TotalCompleted", "iterative": false }, { diff --git a/cartography/data/jobs/analysis/aws_foreign_accounts.json b/cartography/data/jobs/analysis/aws_foreign_accounts.json index fa958a6a0c..b63143b75f 100644 --- a/cartography/data/jobs/analysis/aws_foreign_accounts.json +++ b/cartography/data/jobs/analysis/aws_foreign_accounts.json @@ -2,12 +2,12 @@ "statements": [ { "__comment": "This analyze AWS accounts we created and tag the ones that are foreign. Foreign accounts are ones that were not in the sync scope", - "query": "MATCH (foreign:AWSAccount) where NOT EXISTS(foreign.inscope) SET foreign.foreign = true", + "query": "MATCH (foreign:AWSAccount) where foreign.inscope IS NULL SET foreign.foreign = true", "iterative": false }, { "__comment": "Remove accounts that were set with foreign and inscope. This can happen as we finish the list of sync accounts through assume role mapping and vpc peering", - "query": "MATCH (a:AWSAccount) where EXISTS(a.inscope) AND EXISTS(a.foreign) REMOVE a.foreign", + "query": "MATCH (a:AWSAccount) where a.inscope IS NOT NULL AND a.foreign IS NOT NULL REMOVE a.foreign", "iterative": false } ], diff --git a/cartography/data/jobs/analysis/gcp_compute_asset_inet_exposure.json b/cartography/data/jobs/analysis/gcp_compute_asset_inet_exposure.json index 1b369a4423..04d0d4e33c 100644 --- a/cartography/data/jobs/analysis/gcp_compute_asset_inet_exposure.json +++ b/cartography/data/jobs/analysis/gcp_compute_asset_inet_exposure.json @@ -1,7 +1,7 @@ { "statements": [ { - "query": "MATCH (n) where EXISTS(n.exposed_internet) AND labels(n) IN ['GCPInstance'] WITH n LIMIT $LIMIT_SIZE REMOVE n.exposed_internet, n.exposed_internet_type return COUNT(*) as TotalCompleted", + "query": "MATCH (n) where n.exposed_internet IS NOT NULL AND labels(n) IN ['GCPInstance'] WITH n LIMIT $LIMIT_SIZE REMOVE n.exposed_internet, n.exposed_internet_type return COUNT(*) as TotalCompleted", "iterative": true, "iterationsize": 1000, "__comment__": "Delete exposed_internet off nodes so we can start fresh" @@ -22,17 +22,17 @@ "__comment__": "Delete stale firewall ingress relationships" }, { - "query": "MATCH (ac:GCPNicAccessConfig)<-[:RESOURCE]-(:GCPNetworkInterface)<-[:NETWORK_INTERFACE]-(n:GCPInstance)<-[:FIREWALL_INGRESS]-(firewall_a:GCPFirewall)<-[:ALLOWED_BY]-(allow_rule:GCPIpRule{protocol:'tcp'})<-[:MEMBER_OF_IP_RULE]-(:IpRange{id:\"0.0.0.0/0\"})\nOPTIONAL MATCH (n)<-[:FIREWALL_INGRESS]-(firewall_b:GCPFirewall)<-[:DENIED_BY]-(deny_rule:GCPIpRule{protocol:'tcp'})\nWHERE exists(ac.public_ip) and (\n\tdeny_rule is NULL\n\tOR firewall_b.priority > firewall_a.priority\n\tOR NOT allow_rule.fromport IN RANGE(deny_rule.fromport, deny_rule.toport)\n\tOR NOT allow_rule.toport IN RANGE(deny_rule.fromport, deny_rule.toport)\n)\nSET n.exposed_internet = True, n.exposed_internet_type='direct'\nRETURN count(*) as TotalCompleted", + "query": "MATCH (ac:GCPNicAccessConfig)<-[:RESOURCE]-(:GCPNetworkInterface)<-[:NETWORK_INTERFACE]-(n:GCPInstance)<-[:FIREWALL_INGRESS]-(firewall_a:GCPFirewall)<-[:ALLOWED_BY]-(allow_rule:GCPIpRule{protocol:'tcp'})<-[:MEMBER_OF_IP_RULE]-(:IpRange{id:\"0.0.0.0/0\"})\nOPTIONAL MATCH (n)<-[:FIREWALL_INGRESS]-(firewall_b:GCPFirewall)<-[:DENIED_BY]-(deny_rule:GCPIpRule{protocol:'tcp'})\nWHERE ac.public_ip IS NOT NULL and (\n\tdeny_rule is NULL\n\tOR firewall_b.priority > firewall_a.priority\n\tOR NOT allow_rule.fromport IN RANGE(deny_rule.fromport, deny_rule.toport)\n\tOR NOT allow_rule.toport IN RANGE(deny_rule.fromport, deny_rule.toport)\n)\nSET n.exposed_internet = True, n.exposed_internet_type='direct'\nRETURN count(*) as TotalCompleted", "iterative": false, "__comment__": "Mark a GCP instance with exposed_internet = True and exposed_internet_type = 'direct' if its attached firewalls and TCP rules expose it to the internet." }, { - "query": "MATCH (ac:GCPNicAccessConfig)<-[:RESOURCE]-(:GCPNetworkInterface)<-[:NETWORK_INTERFACE]-(n:GCPInstance)<-[:FIREWALL_INGRESS]-(firewall_a:GCPFirewall)<-[:ALLOWED_BY]-(allow_rule:GCPIpRule{protocol:'udp'})<-[:MEMBER_OF_IP_RULE]-(:IpRange{id:\"0.0.0.0/0\"})\nOPTIONAL MATCH (n)<-[:FIREWALL_INGRESS]-(firewall_b:GCPFirewall)<-[:DENIED_BY]-(deny_rule:GCPIpRule{protocol:'udp'})\nWHERE exists(ac.public_ip) and (\n\tdeny_rule is NULL\n\tOR firewall_b.priority > firewall_a.priority\n\tOR NOT allow_rule.fromport IN RANGE(deny_rule.fromport, deny_rule.toport)\n\tOR NOT allow_rule.toport IN RANGE(deny_rule.fromport, deny_rule.toport)\n)\nSET n.exposed_internet = True, n.exposed_internet_type='direct'\nRETURN count(*) as TotalCompleted", + "query": "MATCH (ac:GCPNicAccessConfig)<-[:RESOURCE]-(:GCPNetworkInterface)<-[:NETWORK_INTERFACE]-(n:GCPInstance)<-[:FIREWALL_INGRESS]-(firewall_a:GCPFirewall)<-[:ALLOWED_BY]-(allow_rule:GCPIpRule{protocol:'udp'})<-[:MEMBER_OF_IP_RULE]-(:IpRange{id:\"0.0.0.0/0\"})\nOPTIONAL MATCH (n)<-[:FIREWALL_INGRESS]-(firewall_b:GCPFirewall)<-[:DENIED_BY]-(deny_rule:GCPIpRule{protocol:'udp'})\nWHERE ac.public_ip IS NOT NULL and (\n\tdeny_rule is NULL\n\tOR firewall_b.priority > firewall_a.priority\n\tOR NOT allow_rule.fromport IN RANGE(deny_rule.fromport, deny_rule.toport)\n\tOR NOT allow_rule.toport IN RANGE(deny_rule.fromport, deny_rule.toport)\n)\nSET n.exposed_internet = True, n.exposed_internet_type='direct'\nRETURN count(*) as TotalCompleted", "iterative": false, "__comment__": "Mark a GCP instance with exposed_internet = True and exposed_internet_type = 'direct' if its attached firewalls and UDP rules expose it to the internet." }, { - "query": "MATCH (ac:GCPNicAccessConfig)<-[:RESOURCE]-(:GCPNetworkInterface)<-[:NETWORK_INTERFACE]-(n:GCPInstance)<-[:FIREWALL_INGRESS]-(firewall_a:GCPFirewall)<-[:ALLOWED_BY]-(allow_rule:GCPIpRule{protocol:'all'})<-[:MEMBER_OF_IP_RULE]-(:IpRange{id:\"0.0.0.0/0\"})\nOPTIONAL MATCH (n)<-[:FIREWALL_INGRESS]-(firewall_b:GCPFirewall)<-[:DENIED_BY]-(deny_rule:GCPIpRule{protocol:'all'})\nWHERE exists(ac.public_ip) and exists(allow_rule.fromport) and exists(allow_rule.toport) and (\n\tdeny_rule is NULL\n\tOR firewall_b.priority > firewall_a.priority\n\tOR NOT allow_rule.fromport IN RANGE(deny_rule.fromport, deny_rule.toport)\n\tOR NOT allow_rule.toport IN RANGE(deny_rule.fromport, deny_rule.toport)\n)\nSET n.exposed_internet = True, n.exposed_internet_type='direct'\nRETURN count(*) as TotalCompleted", + "query": "MATCH (ac:GCPNicAccessConfig)<-[:RESOURCE]-(:GCPNetworkInterface)<-[:NETWORK_INTERFACE]-(n:GCPInstance)<-[:FIREWALL_INGRESS]-(firewall_a:GCPFirewall)<-[:ALLOWED_BY]-(allow_rule:GCPIpRule{protocol:'all'})<-[:MEMBER_OF_IP_RULE]-(:IpRange{id:\"0.0.0.0/0\"})\nOPTIONAL MATCH (n)<-[:FIREWALL_INGRESS]-(firewall_b:GCPFirewall)<-[:DENIED_BY]-(deny_rule:GCPIpRule{protocol:'all'})\nWHERE ac.public_ip IS NOT NULL and allow_rule.fromport IS NOT NULL and allow_rule.toport IS NOT NULL and (\n\tdeny_rule is NULL\n\tOR firewall_b.priority > firewall_a.priority\n\tOR NOT allow_rule.fromport IN RANGE(deny_rule.fromport, deny_rule.toport)\n\tOR NOT allow_rule.toport IN RANGE(deny_rule.fromport, deny_rule.toport)\n)\nSET n.exposed_internet = True, n.exposed_internet_type='direct'\nRETURN count(*) as TotalCompleted", "iterative": false, "__comment__": "Mark a GCP instance with exposed_internet = True and exposed_internet_type = 'direct' if its attached firewalls and ALL rules expose it to the internet." } diff --git a/cartography/data/jobs/analysis/gcp_gke_asset_exposure.json b/cartography/data/jobs/analysis/gcp_gke_asset_exposure.json index 216727ab1c..2768e403c2 100644 --- a/cartography/data/jobs/analysis/gcp_gke_asset_exposure.json +++ b/cartography/data/jobs/analysis/gcp_gke_asset_exposure.json @@ -2,7 +2,7 @@ "statements": [ { "__comment": "This is a clean-up statement to remove custom attributes", - "query": "MATCH (cluster:GKECluster) WHERE EXISTS(cluster.exposed_internet) REMOVE cluster.exposed_internet return COUNT(*) as TotalCompleted", + "query": "MATCH (cluster:GKECluster) WHERE cluster.exposed_internet IS NOT NULL REMOVE cluster.exposed_internet return COUNT(*) as TotalCompleted", "iterative": false }, { diff --git a/cartography/data/jobs/analysis/gcp_gke_basic_auth.json b/cartography/data/jobs/analysis/gcp_gke_basic_auth.json index 71a0f88c6d..da66dd49c7 100644 --- a/cartography/data/jobs/analysis/gcp_gke_basic_auth.json +++ b/cartography/data/jobs/analysis/gcp_gke_basic_auth.json @@ -2,12 +2,12 @@ "statements": [ { "__comment": "This is a clean-up statement to remove custom attributes", - "query": "MATCH (cluster:GKECluster) WHERE EXISTS(cluster.basic_auth) REMOVE cluster.basic_auth return COUNT(*) as TotalCompleted", + "query": "MATCH (cluster:GKECluster) WHERE cluster.basic_auth IS NOT NULL REMOVE cluster.basic_auth return COUNT(*) as TotalCompleted", "iterative": false }, { "__comment": "This sets the basic_auth attribute", - "query": "MATCH (cluster:GKECluster) WHERE (EXISTS(cluster.masterauth_username) AND NOT cluster.masterauth_username = '') AND (EXISTS(cluster.masterauth_password) AND NOT cluster.masterauth.password = '') SET cluster.basic_auth = true", + "query": "MATCH (cluster:GKECluster) WHERE (cluster.masterauth_username IS NOT NULL AND NOT cluster.masterauth_username = '') AND (cluster.masterauth_password IS NOT NULL AND NOT cluster.masterauth.password = '') SET cluster.basic_auth = true", "iterative": false } ], diff --git a/cartography/data/jobs/cleanup/aws_apigateway_details.json b/cartography/data/jobs/cleanup/aws_apigateway_details.json index 9dc793b9a2..fdc7ce9ca6 100644 --- a/cartography/data/jobs/cleanup/aws_apigateway_details.json +++ b/cartography/data/jobs/cleanup/aws_apigateway_details.json @@ -1,7 +1,7 @@ { "statements": [ { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:RestAPI) WHERE EXISTS(s.anonymous_access)\n WITH s LIMIT $LIMIT_SIZE\nREMOVE s.anonymous_access, s.anonymous_actions", + "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:RestAPI) WHERE s.anonymous_access IS NOT NULL\n WITH s LIMIT $LIMIT_SIZE\nREMOVE s.anonymous_access, s.anonymous_actions", "iterative": true, "iterationsize": 100 } diff --git a/cartography/data/jobs/cleanup/aws_kms_details.json b/cartography/data/jobs/cleanup/aws_kms_details.json index b3d50eb981..766fccb198 100644 --- a/cartography/data/jobs/cleanup/aws_kms_details.json +++ b/cartography/data/jobs/cleanup/aws_kms_details.json @@ -1,7 +1,7 @@ { "statements": [ { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:KMSKey) WHERE EXISTS(s.anonymous_access)\n WITH s LIMIT $LIMIT_SIZE\nREMOVE s.anonymous_access, s.anonymous_actions", + "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:KMSKey) WHERE s.anonymous_access IS NOT NULL\n WITH s LIMIT $LIMIT_SIZE\nREMOVE s.anonymous_access, s.anonymous_actions", "iterative": true, "iterationsize": 100 } diff --git a/cartography/data/jobs/cleanup/aws_s3_details.json b/cartography/data/jobs/cleanup/aws_s3_details.json index 19ac65295c..bcd77e43a5 100644 --- a/cartography/data/jobs/cleanup/aws_s3_details.json +++ b/cartography/data/jobs/cleanup/aws_s3_details.json @@ -1,7 +1,7 @@ { "statements": [ { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:S3Bucket) WHERE EXISTS(s.anonymous_access)\n WITH s LIMIT $LIMIT_SIZE\nREMOVE s.anonymous_access, s.anonymous_actions", + "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:S3Bucket) WHERE s.anonymous_access IS NOT NULL\n WITH s LIMIT $LIMIT_SIZE\nREMOVE s.anonymous_access, s.anonymous_actions", "iterative": true, "iterationsize": 100 } diff --git a/cartography/intel/aws/apigateway.py b/cartography/intel/aws/apigateway.py index 7858af4720..828b5c86c5 100644 --- a/cartography/intel/aws/apigateway.py +++ b/cartography/intel/aws/apigateway.py @@ -171,7 +171,7 @@ def _load_apigateway_policies( def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> None: set_defaults = """ MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(restApi:APIGatewayRestAPI) - where NOT EXISTS(restApi.anonymous_actions) + where restApi.anonymous_actions IS NULL SET restApi.anonymous_access = false, restApi.anonymous_actions = [] """ diff --git a/cartography/intel/aws/kms.py b/cartography/intel/aws/kms.py index 9cf451bea3..ec60cb48ba 100644 --- a/cartography/intel/aws/kms.py +++ b/cartography/intel/aws/kms.py @@ -189,7 +189,7 @@ def _load_kms_key_policies(neo4j_session: neo4j.Session, policies: List[Dict], u def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> None: set_defaults = """ - MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(kmskey:KMSKey) where NOT EXISTS(kmskey.anonymous_actions) + MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(kmskey:KMSKey) where kmskey.anonymous_actions IS NULL SET kmskey.anonymous_access = false, kmskey.anonymous_actions = [] """ diff --git a/cartography/intel/aws/s3.py b/cartography/intel/aws/s3.py index d1818a3e03..6524bb4c44 100644 --- a/cartography/intel/aws/s3.py +++ b/cartography/intel/aws/s3.py @@ -359,7 +359,7 @@ def _load_s3_public_access_block( def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> None: set_defaults = """ - MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:S3Bucket) where NOT EXISTS(s.anonymous_actions) + MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:S3Bucket) where s.anonymous_actions IS NULL SET s.anonymous_access = false, s.anonymous_actions = [] """ neo4j_session.run( @@ -368,7 +368,7 @@ def _set_default_values(neo4j_session: neo4j.Session, aws_account_id: str) -> No ) set_encryption_defaults = """ - MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:S3Bucket) where NOT EXISTS(s.default_encryption) + MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(s:S3Bucket) where s.default_encryption IS NULL SET s.default_encryption = false """ neo4j_session.run( diff --git a/docs/root/dev/writing-analysis-jobs.md b/docs/root/dev/writing-analysis-jobs.md index c46798c63b..a31924994e 100644 --- a/docs/root/dev/writing-analysis-jobs.md +++ b/docs/root/dev/writing-analysis-jobs.md @@ -89,7 +89,7 @@ In general, the first statement(s) should be a "clean-up phase" that removes cus { "__comment": "This is a clean-up statement to remove custom attributes", "query": "MATCH (n) - WHERE EXISTS(n.exposed_internet) + WHERE n.exposed_internet IS NOT NULL AND labels(n) IN ['AutoScalingGroup', 'EC2Instance', 'LoadBalancer'] WITH n LIMIT $LIMIT_SIZE REMOVE n.exposed_internet, n.exposed_internet_type diff --git a/docs/root/install.md b/docs/root/install.md index 34a4064788..0e3d8ffd69 100644 --- a/docs/root/install.md +++ b/docs/root/install.md @@ -4,20 +4,31 @@ Time to set up the server that will run Cartography. Cartography _should_ work on both Linux and Windows servers, but bear in mind we've only tested it in Linux so far. Cartography supports Python 3.8. Older versions of Python may work but are not explicitly supported. -1. **Get and install the Neo4j graph database** on your server. - 1. Neo4j requires a JVM (JDK/JRE 11 or higher) to be installed. One option is to install [Amazon Coretto 11](https://docs.aws.amazon.com/corretto/latest/corretto-11-ug/what-is-corretto-11.html). +1. **Run the Neo4j graph database version 4.x** on your server. - ⚠️ Make sure you have `JAVA_HOME` environment variable set. The following works for Mac OS: `export JAVA_HOME=$(/usr/libexec/java_home)` + ⚠️ Neo4j 5.x will probably work but Cartography does not explicitly support it yet. - 1. Go to the [Neo4j download page](https://neo4j.com/download-center/#community), and download Neo4j Community Edition 4.4.\*. + 1. If you prefer **Docker**, follow the Neo4j Docker [official docs](https://github.com/neo4j/docker-neo4j) to run a version 4.x container. - 1. [Install](https://neo4j.com/docs/operations-manual/current/installation/) Neo4j on the server you will run Cartography on. + - If you are using an ARM-based machine like an M1 Mac, you should use an ARM image otherwise performance will be very slow - Neo4j keeps ARM builds [here](https://hub.docker.com/r/arm64v8/neo4j/). - ⚠️ For local testing, you might want to turn off authentication via property `dbms.security.auth_enabled` in file /NEO4J_PATH/conf/neo4j.conf + - If you're just playing around, you can specify the `--env=NEO4J_AUTH=none` argument to your `docker` command to run a Neo4j container without authentication. -1. Configure your data sources. See the configuration section of each relevant intel module for more details. + 1. Else if you prefer a **manual install**, -1. **Get and run Cartography** + 1. Neo4j requires a JVM (JDK/JRE 11 or higher) to be installed. One option is to install [Amazon Coretto 11](https://docs.aws.amazon.com/corretto/latest/corretto-11-ug/what-is-corretto-11.html). + + ⚠️ Make sure you have `JAVA_HOME` environment variable set. The following works for Mac OS: `export JAVA_HOME=$(/usr/libexec/java_home)` + + 1. Go to the [Neo4j download page](https://neo4j.com/download-center/#community), and download Neo4j Community Edition 4.4.\*. If you prefer Docker, you can view Neo4j's instructions [here]. + + 1. [Install](https://neo4j.com/docs/operations-manual/current/installation/) Neo4j on the server you will run Cartography on. + + ⚠️ For local testing, you might want to turn off authentication via property `dbms.security.auth_enabled` in file /NEO4J_PATH/conf/neo4j.conf + +4. Configure your data sources. See the configuration section of each relevant intel module for more details. + +5. **Get and run Cartography** 1. Run `pip install cartography` to install our code. diff --git a/tests/integration/test_basic.py b/tests/integration/test_basic.py index e1943098d6..26e7d12c66 100644 --- a/tests/integration/test_basic.py +++ b/tests/integration/test_basic.py @@ -7,7 +7,7 @@ def test_neo4j_connection(): driver = neo4j.GraphDatabase.driver(settings.get("NEO4J_URL")) with driver.session() as session: - session.run("CALL db.indexes();") + session.run("SHOW INDEXES;") def test_create_indexes(neo4j_session):