diff --git a/.cache/.gitignore b/.cache/.gitignore new file mode 100644 index 0000000000..0db58ef555 --- /dev/null +++ b/.cache/.gitignore @@ -0,0 +1,5 @@ +# This .gitignore is a placeholder so that we can store .cache/ in github. +# We are including a pre-created .cache in Github so that when +# we run docker-compose in WSL2 during dev linting, WSL2 does not attempt +# to create .cache/ with root as the owner. The contents of .cache +# should never be pushed back upstream to the main cartography repo. diff --git a/.env b/.env index a305313ea8..d2f578ad09 100644 --- a/.env +++ b/.env @@ -1,3 +1,4 @@ -# This file is for use with docker compose so that mounting Neo4j volumes doesn't fail with perms errs -GID=10001 -UID=10001 +# This file is for docker-compose dev use so that mounting +# Neo4j volumes doesn't fail with permissions errors. +GID=1000 +UID=1000 diff --git a/Dockerfile b/Dockerfile index 620ccd2410..40023fa72e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ # This is a thin distribution of the cartography software. +# It is published at ghcr.io. FROM python:3.10-slim # the UID and GID to run cartography as diff --git a/dev.Dockerfile b/dev.Dockerfile index eef8fea6ad..a1c9a5585b 100644 --- a/dev.Dockerfile +++ b/dev.Dockerfile @@ -1,28 +1,33 @@ -# Builds cartography container for development by performing a Python editable install of the current source code. +# This image is for dev only. +# Performs a Python editable install of the current Cartography source. +# Assumptions: +# - This dockerfile will get called with .cache as a volume mount. +# - The current working directory on the host building this container +# is the cartography source tree from github. FROM python:3.10-slim -# the UID and GID to run cartography as -# (https://github.com/hexops/dockerfile#do-not-use-a-uid-below-10000). -ARG uid=10001 -ARG gid=10001 +# The UID and GID to run cartography as. +# This needs to match the gid and uid on the host. +# Update this to match. On WSL2 this is usually 1000. +ARG uid=1000 +ARG gid=1000 RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends make git && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -# Assumption: current working directory is the cartography source tree from github. -COPY . /var/cartography +# Install dependencies. WORKDIR /var/cartography -ENV HOME=/var/cartography +COPY . /var/cartography +RUN pip install -r test-requirements.txt && \ + pip install -U -e . && \ + chmod -R a+w /var/cartography -RUN pip install -U -e . && \ - pip install -r test-requirements.txt && \ - # Grant write access to the directory for unit and integration test coverage files - chmod -R a+w /var/cartography && \ - # Sets the directory as safe due to a mismatch in the user that cloned the repo - # and the user that is going to run the unit&integ tests. This lets pre-commit work. - git config --global --add safe.directory /var/cartography && \ +# Now copy the entire source tree. +ENV HOME=/var/cartography +# Necessary for pre-commit. +RUN git config --global --add safe.directory /var/cartography && \ git config --local user.name "cartography" USER ${uid}:${gid} diff --git a/docker-compose.yml b/docker-compose.yml index 105fbb0c8b..1353555330 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,6 @@ services: - ./.compose/neo4j/import:/import - ./.compose/neo4j/logs:/logs - ./.compose/neo4j/plugins:/plugins - user: "${UID}:${GID}" environment: # Raise memory limits: - NEO4J_dbms_memory_pagecache_size=1G @@ -34,14 +33,15 @@ services: # Networking: - dbms.connector.bolt.listen_address=0.0.0.0:7687 healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:7474"] + test: ["CMD", "wget", "--no-verbose", "http://localhost:7474"] interval: 10s timeout: 10s retries: 10 # Runs the standard cartography image available at ghcr.io. cartography: - image: ghcr.io/lyft/cartography:latest + image: ghcr.io/cartography-cncf/cartography:latest + platform: linux/x86_64 # EXAMPLE: Our ENTRYPOINT is cartography, running specific command to sync AWS # command: ["-v", "--neo4j-uri=bolt://neo4j:7687", "--aws-sync-all-profiles"] init: true @@ -50,6 +50,7 @@ services: - neo4j volumes: # Provide AWS creds to the container + # Add other volumes here to support other data providers. - ~/.aws:/var/cartography/.aws/ environment: # Point to the neo4j service defined in this docker-compose file. @@ -58,9 +59,8 @@ services: # Intended to run local automated tests, custom sync scripts, and local changes. cartography-dev: # See dev instructions: we assume that you have built this with - # `docker build -t lyft/cartography-dev . -f dev.Dockerfile`. - # Do not push this image remotely! - image: lyft/cartography-dev + # `docker build -t cartography-cncf/cartography-dev -f dev.Dockerfile ./` + image: cartography-cncf/cartography-dev init: true restart: on-failure depends_on: @@ -71,6 +71,10 @@ services: # For pre-commit to work - .:/var/cartography - ./.cache/pre-commit:/var/cartography/.cache/pre-commit + # for git, for precommit + - ./.git:/var/cartography/.git environment: # Point to the neo4j service defined in this docker-compose file. - NEO4J_URL=bolt://cartography-neo4j-1:7687 + # this is actually needed + - PRE_COMMIT_HOME=/var/cartography/.cache diff --git a/docs/root/dev/developer-guide.md b/docs/root/dev/developer-guide.md index b927d292e7..53c30c0a99 100644 --- a/docs/root/dev/developer-guide.md +++ b/docs/root/dev/developer-guide.md @@ -99,8 +99,7 @@ We include a dev.Dockerfile that can help streamline common dev tasks. It is dif To use it, build dev.Dockerfile with ```bash cd /path/to/cartography/repo -docker build -t lyft/cartography-dev -f . dev.Dockerfile -docker-compose --profile dev up -d +docker build -t cartography-cncf/cartography-dev -f dev.Dockerfile ./ ``` With that, there are some interesting things you can do with it. @@ -137,7 +136,7 @@ If you don't like docker-compose or if it doesn't work for you for any reason, h #### Run unit tests with dev.Dockerfile ```bash -docker run --rm lyft/cartography-dev make test_unit +docker run --rm cartography-cncf/cartography-dev make test_unit ``` This is a simple command because it doesn't require any volume mounts or docker networking. @@ -148,7 +147,7 @@ This is a simple command because it doesn't require any volume mounts or docker docker run --rm \ -v $(pwd):/var/cartography \ -v $(pwd)/.cache/pre-commit:/var/cartography/.cache/pre-commit \ - lyft/cartography-dev \ + cartography-cncf/cartography-dev \ make test_lint ``` @@ -173,7 +172,7 @@ and then call the integration test suite like this: docker run --rm \ --network cartography-network \ -e NEO4J_URL=bolt://cartography-neo4j:7687 \ - lyft/cartography-dev \ + cartography-cncf/cartography-dev \ make test_integration ``` @@ -200,20 +199,20 @@ docker run --rm \ -v $(pwd)/.cache/pre-commit:/var/cartography/.cache/pre-commit \ --network cartography-network \ -e NEO4J_URL=bolt://cartography-neo4j:7687 \ - lyft/cartography-dev \ + cartography-cncf/cartography-dev \ make test ``` #### Run a [custom sync script](#implementing-custom-sync-commands) with dev.Dockerfile ```bash -docker run --rm lyft/cartography-dev python custom_sync.py +docker run --rm cartography-cncf/cartography-dev python custom_sync.py ``` #### Run cartography CLI with dev.Dockerfile ```bash -docker run --rm lyft/cartography-dev cartography --help +docker run --rm cartography-cncf/cartography-dev cartography --help ``` ## How to write a new intel module diff --git a/docs/root/install.md b/docs/root/install.md index 4677da51fa..0e9e41e752 100644 --- a/docs/root/install.md +++ b/docs/root/install.md @@ -124,7 +124,7 @@ Read on to see [other things you can do with Cartography](#things-to-do-next). -v ~/.aws:/var/cartography/.aws/ \ -e AWS_PROFILE=1234_testprofile \ -e AWS_DEFAULT_REGION=us-east-1 \ - lyft/cartography --neo4j-uri bolt://cartography-neo4j:7687 + cartography-cncf/cartography --neo4j-uri bolt://cartography-neo4j:7687 ``` If things work, your terminal will look like this where you see log messages displaying how many assets are being loaded to the graph: @@ -139,7 +139,7 @@ Read on to see [other things you can do with Cartography](#things-to-do-next). - `AWS_DEFAULT_REGION` must be specified. - Our docker-compose.yml maps in `~/.aws/` on your host machine to `/var/cartography/.aws` in the cartography container, so the container has access to AWS profile and credential files. - - You can view a full list of Cartography's CLI arguments by running `docker run lyft/cartography --help`. + - You can view a full list of Cartography's CLI arguments by running `docker run cartography-cncf/cartography --help`. 1. **View the graph.** diff --git a/tests/unit/cartography/intel/github/test_github.py b/tests/unit/cartography/intel/github/test_github.py index 0a2ffd5c05..ff213a32be 100644 --- a/tests/unit/cartography/intel/github/test_github.py +++ b/tests/unit/cartography/intel/github/test_github.py @@ -16,11 +16,13 @@ from tests.data.github.rate_limit import RATE_LIMIT_RESPONSE_JSON +@patch('cartography.intel.github.util.time.sleep') @patch('cartography.intel.github.util.handle_rate_limit_sleep') @patch('cartography.intel.github.util.fetch_page') def test_fetch_all_handles_retries( mock_fetch_page: Mock, mock_handle_rate_limit_sleep: Mock, + mock_sleep: Mock, ) -> None: ''' Ensures that fetch_all re-reaises the same exceptions when exceeding retry limit