diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c1a53b6 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,48 @@ +name: CI Tests + +on: + + #NOTE: All jobs gated by auth job + + #Regular dev + push: + pull_request: + + #Enable UI-driven branch testing + workflow_dispatch: + + #Test main bidaily @ 1a + schedule: + - cron: '0 1 1-31/2 * *' + +jobs: + + test-docs: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Test building docs + run: | + ./bin/ci.sh + + test-readme: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.8 + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + - name: Test building docs + continue-on-error: true + run: | + docker run --rm -v "$(pwd)/README.md:/workdir/README.md:ro" -v "$(pwd)/.markdownlint.yaml:/workdir/.markdownlint.yaml:ro" ghcr.io/igorshubovych/markdownlint-cli:v0.37.0 README.md + + \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b92ce54 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +_build/ +doctrees/ diff --git a/.markdownlint.yml b/.markdownlint.yml new file mode 100644 index 0000000..0279472 --- /dev/null +++ b/.markdownlint.yml @@ -0,0 +1,267 @@ +# ------------------------------------------------------------------------------ +# Based on markdownlint/schema/.markdownlint.yml +# ------------------------------------------------------------------------------ + + +# Example markdownlint configuration with all properties set to their default value + +# Default state for all rules +default: true + +# Path to configuration file to extend +extends: null + +# MD001/heading-increment : Heading levels should only increment by one level at a time : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md001.md +MD001: true + +# MD003/heading-style : Heading style : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md003.md +MD003: + # Heading style + style: "consistent" + +# MD004/ul-style : Unordered list style : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md004.md +MD004: + # List style + style: "consistent" + +# MD005/list-indent : Inconsistent indentation for list items at the same level : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md005.md +MD005: true + +# MD007/ul-indent : Unordered list indentation : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md007.md +MD007: + # Spaces for indent + indent: 2 + # Whether to indent the first level of the list + start_indented: false + # Spaces for first level indent (when start_indented is set) + start_indent: 2 + +# MD009/no-trailing-spaces : Trailing spaces : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md009.md +MD009: + # Spaces for line break + br_spaces: 2 + # Allow spaces for empty lines in list items + list_item_empty_lines: false + # Include unnecessary breaks + strict: false + +# MD010/no-hard-tabs : Hard tabs : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md010.md +MD010: + # Include code blocks + code_blocks: true + # Fenced code languages to ignore + ignore_code_languages: [] + # Number of spaces for each hard tab + spaces_per_tab: 1 + +# MD011/no-reversed-links : Reversed link syntax : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md011.md +MD011: true + +# MD012/no-multiple-blanks : Multiple consecutive blank lines : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md012.md +MD012: + # Consecutive blank lines + maximum: 1 + +# MD013/line-length : Line length : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md013.md +MD013: + # Number of characters + line_length: 2000 + # Number of characters for headings + heading_line_length: 2000 + # Number of characters for code blocks + code_block_line_length: 2000 + # Include code blocks + code_blocks: true + # Include tables + tables: false + # Include headings + headings: true + # Strict length checking + strict: false + # Stern length checking + stern: false + +# MD014/commands-show-output : Dollar signs used before commands without showing output : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md014.md +MD014: true + +# MD018/no-missing-space-atx : No space after hash on atx style heading : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md018.md +MD018: true + +# MD019/no-multiple-space-atx : Multiple spaces after hash on atx style heading : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md019.md +MD019: true + +# MD020/no-missing-space-closed-atx : No space inside hashes on closed atx style heading : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md020.md +MD020: true + +# MD021/no-multiple-space-closed-atx : Multiple spaces inside hashes on closed atx style heading : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md021.md +MD021: true + +# MD022/blanks-around-headings : Headings should be surrounded by blank lines : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md022.md +MD022: + # Blank lines above heading + lines_above: 1 + # Blank lines below heading + lines_below: 1 + +# MD023/heading-start-left : Headings must start at the beginning of the line : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md023.md +MD023: true + +# MD024/no-duplicate-heading : Multiple headings with the same content : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md024.md +MD024: + # Only check sibling headings + allow_different_nesting: false + # Only check sibling headings + siblings_only: false + +# MD025/single-title/single-h1 : Multiple top-level headings in the same document : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md025.md +MD025: + # Heading level + level: 1 + # RegExp for matching title in front matter + front_matter_title: "^\\s*title\\s*[:=]" + +# MD026/no-trailing-punctuation : Trailing punctuation in heading : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md026.md +MD026: + # Punctuation characters + punctuation: ".,;:!。,;:!" + +# MD027/no-multiple-space-blockquote : Multiple spaces after blockquote symbol : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md027.md +MD027: true + +# MD028/no-blanks-blockquote : Blank line inside blockquote : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md028.md +MD028: true + +# MD029/ol-prefix : Ordered list item prefix : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md029.md +MD029: + # List style + style: "one_or_ordered" + +# MD030/list-marker-space : Spaces after list markers : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md030.md +MD030: + # Spaces for single-line unordered list items + ul_single: 1 + # Spaces for single-line ordered list items + ol_single: 1 + # Spaces for multi-line unordered list items + ul_multi: 1 + # Spaces for multi-line ordered list items + ol_multi: 1 + +# MD031/blanks-around-fences : Fenced code blocks should be surrounded by blank lines : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md031.md +MD031: + # Include list items + list_items: true + +# MD032/blanks-around-lists : Lists should be surrounded by blank lines : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md032.md +MD032: true + +# MD033/no-inline-html : Inline HTML : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md033.md +MD033: + # Allowed elements + allowed_elements: [table, tr, td, img, em,br, a] + +# MD034/no-bare-urls : Bare URL used : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md034.md +MD034: true + +# MD035/hr-style : Horizontal rule style : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md035.md +MD035: + # Horizontal rule style + style: "consistent" + +# MD036/no-emphasis-as-heading : Emphasis used instead of a heading : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md036.md +MD036: + # Punctuation characters + punctuation: ".,;:!?。,;:!?" + +# MD037/no-space-in-emphasis : Spaces inside emphasis markers : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md037.md +MD037: true + +# MD038/no-space-in-code : Spaces inside code span elements : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md038.md +MD038: true + +# MD039/no-space-in-links : Spaces inside link text : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md039.md +MD039: true + +# MD040/fenced-code-language : Fenced code blocks should have a language specified : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md040.md +MD040: + # List of languages + allowed_languages: [] + # Require language only + language_only: false + +# MD041/first-line-heading/first-line-h1 : First line in a file should be a top-level heading : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md041.md +MD041: + # Heading level + level: 1 + # RegExp for matching title in front matter + front_matter_title: "^\\s*title\\s*[:=]" + +# MD042/no-empty-links : No empty links : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md042.md +MD042: true + +# MD043/required-headings : Required heading structure : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md043.md +MD043: false + +# MD044/proper-names : Proper names should have the correct capitalization : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md044.md +MD044: + # List of proper names + names: [] + # Include code blocks + code_blocks: true + # Include HTML elements + html_elements: true + +# MD045/no-alt-text : Images should have alternate text (alt text) : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md045.md +MD045: true + +# MD046/code-block-style : Code block style : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md046.md +MD046: + # Block style + style: "consistent" + +# MD047/single-trailing-newline : Files should end with a single newline character : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md047.md +MD047: true + +# MD048/code-fence-style : Code fence style : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md048.md +MD048: + # Code fence style + style: "consistent" + +# MD049/emphasis-style : Emphasis style : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md049.md +MD049: + # Emphasis style + style: "consistent" + +# MD050/strong-style : Strong style : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md050.md +MD050: + # Strong style + style: "consistent" + +# MD051/link-fragments : Link fragments should be valid : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md051.md +MD051: true + +# MD052/reference-links-images : Reference links and images should use a label that is defined : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md052.md +MD052: + # Include shortcut syntax + shortcut_syntax: false + +# MD053/link-image-reference-definitions : Link and image reference definitions should be needed : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md053.md +MD053: + # Ignored definitions + ignored_definitions: + - "//" + +# MD054/link-image-style : Link and image style : https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md054.md +MD054: + # Allow autolinks + autolink: true + # Allow inline links and images + inline: true + # Allow full reference links and images + full: true + # Allow collapsed reference links and images + collapsed: true + # Allow shortcut reference links and images + shortcut: true + # Allow URLs as inline links + url_inline: true \ No newline at end of file diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..33fc5cc --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,59 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.12" + apt_packages: + # More closely mirror https://github.com/sphinx-doc/sphinx-docker-images + - graphviz + - imagemagick + - make + - pandoc + - texlive-latex-base + - texlive-latex-recommended + - texlive-latex-extra + - texlive-fonts-recommended + commands: + + # setup + - pip install "graphistry[docs]" + - pip install -r infra/requirements-python.txt + #- cp -r demos docs/source/demos + - cp README.md docs/README.md + + # build html + - sphinx-build -b html -d doctrees docs $READTHEDOCS_OUTPUT/html/ + + # build epub + - sphinx-build -b epub -d doctrees docs _build/latexpdf + - mkdir -p $READTHEDOCS_OUTPUT/epub + - ls _build/latexpdf + - ls $READTHEDOCS_OUTPUT/epub + - cp _build/latexpdf/GraphistryCLI.epub $READTHEDOCS_OUTPUT/epub/GraphistryCLI.epub + + # build pdf + - sphinx-build -b latex -d doctrees docs _build/latexpdf + - ls _build/latexpdf + - cd _build/latexpdf && (pdflatex -file-line-error -interaction=nonstopmode Graphistry.tex || test -f _build/latexpdf/Graphistry.pdf) && (pdflatex -file-line-error -interaction=nonstopmode Graphistry.tex || test -f _build/latexpdf/Graphistry.pdf) && echo ok || { echo fail && exit 1 ; } + - mkdir -p $READTHEDOCS_OUTPUT/pdf + - cp _build/latexpdf/Graphistry.pdf $READTHEDOCS_OUTPUT/pdf/Graphistry.pdf + + +#for nav links? +formats: + - pdf + - epub + - htmlzip + +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..5184fac --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,21 @@ +# Changelog + +All notable changes to this repo are documented in this file. The Graphistry server is tracked in the main [Graphistry major release history documentation](https://graphistry.zendesk.com/hc/en-us/articles/360033184174-Enterprise-Release-List-Downloads). + +The changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and all PyGraphistry-specific breaking changes are explictly noted here. + +## [Development] + +### Added + +* ReadTheDocs site: [https://graphistry-admin-docs.readthedocs.io/](https://graphistry-admin-docs.readthedocs.io/) + +### Changed + +* Reorganized documentation into thematic areas + +### Infra + +* Sphinx port +* CI diff --git a/README.md b/README.md index 02a4790..d1e221e 100644 --- a/README.md +++ b/README.md @@ -2,122 +2,33 @@ Graphistry is the most scalable graph-based visual analysis and investigation automation platform. It supports both cloud and on-prem deployment options. Big graphs are tons of fun! -The documentation here covers system administration -See bottom of page for table of contents and additional resources. +## Quick administration links -#### Get +* [Top commands](https://graphistry-admin-docs.readthedocs.io/en/latest/commands.html) +* [Plan deployments](https://graphistry-admin-docs.readthedocs.io/en/latest/planning/hardware-software.html) +* Install: [Cloud](https://graphistry-admin-docs.readthedocs.io/en/latest/install/cloud/index.html) & [On-prem](https://graphistry-admin-docs.readthedocs.io/en/latest/install/on-prem/index.html) +* [Configure](https://graphistry-admin-docs.readthedocs.io/en/latest/app-config/index.html) +* [Debugging & performance](https://graphistry-admin-docs.readthedocs.io/en/latest/debugging/index.html) +* [Security](https://graphistry-admin-docs.readthedocs.io/en/latest/security/index.html) +* [Operations & tools](https://graphistry-admin-docs.readthedocs.io/en/latest/tools/index.html) +* [FAQ](https://graphistry-admin-docs.readthedocs.io/en/latest/faq/index.html) & [support options](https://graphistry-admin-docs.readthedocs.io/en/latest/support.html) -Pick an [appropriate hardware/software configuration](hardware-software.md): -* Graphistry Hub: Graphistry manages Hub for its users -* AWS/Azure Marketplace: See below instructions -* Docker (self-hosted): See [enterprise release portal](https://graphistry.zendesk.com/hc/en-us/articles/360033184174) -* Experimental - Kubernetes Helm chart: See [graphistry/graphistry-helm repository](https://github.com/graphistry/graphistry-helm) - -#### Support - -Email, Zoom, Slack, phone, tickets -- we encourage using the [Graphistry support channel](https://www.graphistry.com/support) that works best for you. We want you and your users to succeed! - -#### Learn - -See also [release guides](https://github.com/graphistry/graphistry-community) and [user and developer documentation](https://hub.graphistry.com/docs) - ---- - -## Quick launch: Managed - -[AWS and Azure marketplaces](https://www.graphistry.com/get-started): The fastest way to start using Graphistry is to quick launch a preconfigured private Graphistry instance on [AWS and Azure marketplaces](https://www.graphistry.com/get-started) - -It runs in your private cloud provider account and is configured to autostart. See [AWS launch walkthrough tutorial & videos](https://www.graphistry.com/blog/marketplace-tutorial) and linked guides for optional post-launch configuration and maintenance. - -Out-of-the-box configurations include GPU drivers, Docker with Nvidia runtime, multi-GPU support, Graphistry installation, auto-start on instance stop/restart, and more - ---- - -## Quick launch: Manual - -Requirements: [Download Graphistry](https://graphistry.zendesk.com/hc/en-us/articles/360033184174) and [verify docker-compose is setup for Nvidia runtimes](docs/testing-an-install.md#6-quick-testing-and-test-gpu) - -**1. Install** if not already available from the folder with `containers.tar.gz`, and likely using `sudo`: - -```bash -docker load -i containers.tar.gz -``` - -Note: In previous versions (< `v2.35`), the file was `containers.tar` - - -**2. Launch** from the folder with `docker-compose.yml` if not already up, and likely using `sudo`: - -```bash -docker-compose up -d -``` - -Note: Takes 1-3 min, and around 5 min, `docker ps` should report all services as `healthy` - -**3. Test:** Go to - -``` -http://localhost -``` - -* Create an account, and then try running a prebuilt Jupyter Notebook from the dashboard! - * The first account gets an admin role, upon which account self-registration closes. Admins can then invite users or open self-registration. See [User Creation](docs/user-creation.md) for more information. - -* Try a visualization like http://localhost/graph/graph.html?dataset=Facebook&play=5000&splashAfter=false - * **Warning**: First viz load may be slow (1 min) as RAPIDS generates **just-in-time** code for each GPU worker upon first encounter, and/or require a page refresh - ---- - - -## Top commands - -Graphistry supports advanced command-line administration via standard `docker-compose`, `.yml` / `.env` files, and `caddy` reverse-proxy configuration. - -### Login to server - -| Image | Command | -|--: |:-- | -| **AWS** | `ssh -i [your_key].pem ubuntu@[your_public_ip]` | -| **Azure** | `ssh -i [your_key].pem [your_user]@[your_public_ip]`
`ssh [your_user]@[your_public_ip]` (pwd-based) | -| **Google** | `gcloud compute [your_instance] ssh` | -| **On-prem / BYOL** | Contact your admin | - -### CLI commands - -All likely require `sudo`. Run from where your `docker-compose.yml` file is located: `/home/ubuntu/graphistry` (AWS), `/var/graphistry` (Azure), or `/var/graphistry//` (recommended on-prem). - -| TASK | COMMAND | NOTES | -|--: |:--- |:--- | -| **Install** | `docker load -i containers.tar.gz` | Install the `containers.tar.gz` Graphistry release from the current folder. You may need to first run `tar -xvvf my-graphistry-release.tar.gz`. | -| **Start
interactive** | `docker-compose up` | Starts Graphistry, close with ctrl-c | -| **Start
daemon** | `docker-compose up -d` | Starts Graphistry as background process | -| **Start
namespaced (concurrent)** | `docker-compose -p my_unique_namespace up` | Starts Graphistry in a specific namespace. Enables running multiple independent instances of Graphistry. NOTE: Must modify Caddy service in `docker-compose.yml` to use non-conflicting public ports, and likewise change global volumes to be independent. | -| **Stop** | `docker-compose stop` | Stops Graphistry | -| **Restart (soft)** | `docker restart ` | Soft restart. May also need to restart service `nginx`. | -| **Restart (hard)** | `docker up -d --force-recreate --no-deps ` | Restart with fresh state. May also need to restart service `nginx`. | -| **Reset** | `docker-compose down -v && docker-compose up -d` | Stop Graphistry, remove all internal state (including the user account database!), and start fresh . | -| **Status** | `docker-compose ps`, `docker ps`, and `docker status` | Status: Uptime, healthchecks, ... | -| **GPU Status** | `nvidia-smi` | See GPU processes, compute/memory consumption, and driver. Ex: `watch -n 1.5 nvidia-smi`. Also, `docker run --rm -it nvidia/cuda:11.5.0-base-ubuntu20.04 nvidia-smi` for in-container test. | -| **1.0 API Key** | docker-compose exec streamgl-vgraph-etl curl "http://0.0.0.0:8080/api/internal/provision?text=MYUSERNAME" | Generates API key for a developer or notebook user (1.0 API is deprecated)| -| **Logs** | `docker-compose logs ` | Ex: Watch all logs, starting with the 20 most recent lines: `docker-compose logs -f -t --tail=20 forge-etl-python` . You likely need to switch Docker to use the local json logging driver by deleting the two default managed Splunk log driver options in `/etc/docker/daemon.json` and then restarting the `docker` daemon (see below). | -| **Create Users** | Use Admin Panel (see [Create Users](docs/user-creation.md)) or `etc/scripts/rest` | -| **Restart Docker Daemon** | `sudo service docker restart` | Use when changing `/etc/docker/daemon.json`, ... | -| **Jupyter shell**| `docker exec -it -u root graphistry_notebook_1 bash` then `source activate rapids` | Use for admin tasks like global package installs | - - -## Manual enterprise install - -NOTE: Managed Graphistry instances do not require any of these steps. +## Further reading -The Graphistry environnment depends soley on [Nvidia RAPIDS](https://rapids.ai) and [Nvidia Docker](https://github.com/NVIDIA/nvidia-docker) via `Docker Compose 3`, and ships with all other dependencies built in. +* [Main Graphistry documentation](https://hub.graphistry.com/docs) and same path on your local server +* [Release portal](https://graphistry.zendesk.com/hc/en-us/articles/360033184174) for enterprise admins to download the latest +* [Release notes](https://graphistry.zendesk.com/hc/en-us/articles/360033184174) +* [Graphistry Hub](https://hub.graphistry.com): Graphistry-managed GPU servers, including free and team tiers +* Docker (self-hosted): See [enterprise release portal](https://graphistry.zendesk.com/hc/en-us/articles/360033184174) +* [Kubernetes Helm charts](https://github.com/graphistry/graphistry-helm) - Experimental -### Test your environment setup +## Quick GPU Docker environment test You can test your GPU environment via Graphistry's [base RAPIDS Docker image on DockerHub](https://hub.docker.com/r/graphistry/graphistry-forge-base): -``` +```bash docker run --rm -it --entrypoint=/bin/bash graphistry/graphistry-forge-base:latest -c "source activate rapids && python3 -c \"import cudf; print(cudf.DataFrame({'x': [0,1,2]})['x'].sum())\"" ``` @@ -126,122 +37,6 @@ docker run --rm -it --entrypoint=/bin/bash graphistry/graphistry-forge-base:late 3 ``` -See [GPU testing](docs/testing-an-install.md#6-quick-testing-and-test-gpu) to identify individual issues. - -### Manual environment setup - -See sample [Ubuntu 18.04 TLS](./docs/ubuntu_18_04_lts_setup.md) and [RHEL 7.6](./docs/rhel_7_6_setup.md) environment setup scripts for Nvidia drivers, Docker, nvidia-docker runtime, and docker-compose. See [Testing an Install](docs/testing-an-install.md) for environment testing. - -Please contact Graphistry staff for environment automation options. - -### Manual Graphistry container download - -Download the latest enterprise distribution from the [enterprise release list](https://graphistry.zendesk.com/hc/en-us/articles/360033184174-Enterprise-Releases) on the support site. Please contact your Graphistry support staff for access if not available. - -### Manual Graphistry container installation - -If `nvidia` is already your `docker info | grep Default` runtime: - -```bash -############ Install & Launch -wget -O release.tar.gz "https://..." -tar -xvvf release.tar.gz -docker load -i containers.tar.gz -docker-compose up -d -``` - - -## FAQ - -* Where are the docs? See this [GitHub repository](https://github.com/graphistry/graphistry-cli) for admin docs and [Graphistry Hub docs](http://hub.graphistry.com/docs) (or http://your_graphistry/docs) for analyst and developer docs - -* Where do I get help? Whether community chat, email, tickets, a call, or even a training, [pick the most convienent option](https://www.graphistry.com/support) - -* Can Graphistry run in the cloud? Yes - privately both via a [preconfigured AWS/Azure marketplace](https://www.graphistry.com/get-started) and as a self-managed docker binary. Contact our team for upcoming managed Graphistry Hub cloud tiers. - -* Can Graphistry run privately? - * On-prem, including air-gapped, as a team backend server or a Linux-based analyst workstation, via docker image - * Cloud, via prebuilt marketplace instance - * Cloud, via docker image - -* Can Graphistry run in ... - * A VM: [Yes, including VMWare vSphere, Nutanix AHV, and anywhere else Nvidia RAPIDS.ai runs](docs/vGPU.md). Just set `RMM_ALLOCATOR=default` in your `data/config/custom.env` to avoid relying on CUDA Unified Memory, which vGPUs do not support. - * Ubuntu / Red Hat / ... : Yes, just ensure the Nvidia Docker runtime is set as the default for docker-compose. We can assist with reference environment bootstrap scripts. - * Podman: Maybe! We have confirmed our core containers run on RHEL 8.3 with Podman, Nvidia container runtime, and docker-compose cli. Please contact our staff for the possibility of an alternate podman-compatible tarball. - -* How do I do license management? - * Graphistry does not require software-managed license registration, we can work with your procurement team on self-reported use - -* Do I need a GPU on the client? No, clients do not need a GPU. They do need WebGL enabled, such as Chrome's non-GPU software emulation mode. If some of your users are on extremely limited environments, e.g., worse than a modern phone, or you have extremely powerful GPUs you would like to share, users report great experiences with GPU VDI technologies. - -* Do I need a GPU on the server? Yes, the server requires an Nvidia GPU that is Pascal or later (T4, P100, V100, A100, RTX, ...). - -* Can Graphistry use multiple GPUs and multiple servers? - * Graphistry visualizations take advantage of multiple GPUs & CPUs on the same server to handle more users - * Graphistry-managed Jupyter notebooks enable users to run custom GPU code, where each user may run multi-GPU tasks (e.g., via dask-cudf and dask-sql) - * For high availability configuration and operation, contact staff for additional guidance - * For many-node deployment and multi-GPU visualization acceleration, contact staff for roadmap - -* Can I run multiple instances of Graphistry? Yes, see the command section for running in an isolated namespace. This is primarily for testing and in-place upgrading. If your goal is for Graphistry to use multiple CPUs and GPUs, it already does so automatically, so you can launch as usual. - -* Can I use Graphistry from OS X / Windows? Yes, analysts can use any modern browser on any modern OS such as Chrome on Windows and Firefox on OS X, and even on small devices like Android phones and Apple tablets. The server requires Linux (Ubuntu, RHEL, ...) with a GPU. A self-contained analyst workstation would be Linux based. - -* How do I try it out? - * Notebook/API users can get a free account on [Graphistry Hub](https://www.graphistry.com/get-started) - * Interact with pregenerated live visualizations on the [PyGraphistry gallery](https://github.com/graphistry/pygraphistry) - * If you have a private sample CSV/XLS/etc., you can [spin up a private server in your AWS/Azure account](https://www.graphistry.com/get-started) and turn it off when done, and [our team is happy to help](https://www.graphistry.com/support) - -* How can I test if my GPU supports Graphistry and my GPU environment is setup properly? Graphistry only requires a [RAPIDS-compatible](https://www.rapids.ai) Docker environment, so you can use the community resources for that. In addition, see [Testing an Install](docs/testing-an-install.md). - -* The server is slow to start, is it broken? - * The server may take 1-3min to start; check the health status of each service with `sudo docker ps` - * By default, Graphistry has 4 RAPIDS workers (service `etl-server-python`) that perform just-in-time GPU compilation, meaning the first load on each is slow. - * ... System start and the first visualization load per process might be sped up by ensuring Docker is using a native diff driver (see [performance tuning](docs/performance-tuning.md)) - * ... Subsequent use of those workers are fast for new datasets (code is already compiled), and subsequent reloads of recent datasets are extra fast (cached) - -* Can I add extra security layers? Yes -- see the hardening section for configuring areas like TLS, and contact the team for assistance with more custom/experimental layers like SSO - -* Can I run on another port? Yes -- modify `docker-compose.yml`'s service `caddy:`, such as for 80 to instead be 8888: -```yml - ports: - - 8888:80 - expose: - - "8888" -``` - - -## Further reading +See the installation and debugging sections for additional scenarios such as ensuring Docker Compose is correctly defaulting to a GPU runtime. -* Plan a deployment - * Architecture: [Deployment architecture planning guide](docs/deployment-planning.md) - * Capacity: [Hardware/software requirements](hardware-software.md) - * Security: [Recommended hardening](docs/configure-security.md), [threat model](docs/threatmodel.md), [authentication](docs/authentication.md) -* Install - * [Release portal](https://graphistry.zendesk.com/hc/en-us/articles/360033184174) for enterprise admins to download the latest Docker images - * [Manual Graphistry Installation](docs/manual.md) for non-marketplace / non-hosted: -
AWS BYOL, Azure BYOL, On-Prem (RHEL/Ubuntu), & Air-Gapped - * [Nvidia vGPU virtualization support](docs/vGPU.md) -* Configure - * [System settings](docs/configure.md): -
TLS/SSL/HTTPS, performance, logging, backups to disk, multiple proxy layers, and more - * [Security: Enable auto-TLS and restrict network access](docs/configure-security.md) - * [Add users](docs/user-creation.md) - * [AWS Marketplace quickstart](docs/aws_marketplace.md) - * [Azure Marketplace quickstart](docs/azure_marketplace.md) - also [private BYOL plans](docs/azure_marketplace_private_offer.md) and [manual VM setup](docs/azure.md) (deprecated) - * [Dashboards](docs/configure-dashboards.md) - * Content: - * [Investigations connectors](docs/configure-investigation.md): Splunk, Neo4j, and more - * [Investigation templates](docs/templates.md): Save, reuse, link, and embed workflows - * [Custom pivots](docs/configure-custom-pivots.md): Streamline common investigation steps with simplified UIs - * [Ontology](docs/configure-ontology.md): Add new types and customize colors, icons, sizes, and more - * The [Graphistry Data Bridge](docs/bridge.md): Go between cloud <> on-prem - * [PyGraphistry and notebooks](docs/configure-pygraphistry.md) -* Maintain - * [Roadmap](https://github.com/graphistry/graphistry-community/projects/1) and [release notes](https://graphistry.zendesk.com/hc/en-us/articles/360033184174) - * On unconfigured instance reboots, you may need to first run `sudo systemctl start docker` - * [Update, backup, and migrate](docs/update-backup-migrate.md) -* Use and develop - * [Main documentation](https://hub.graphistry.com/docs) or the same URL adjusted for your local server's address -* Test: - * [Testing an install](docs/testing-an-install.md) diff --git a/bin/ci.sh b/bin/ci.sh new file mode 100755 index 0000000..1597ebc --- /dev/null +++ b/bin/ci.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -ex + +DOCS_FORMAT=${DOCS_FORMAT:-all} # Default to building all formats if not specified + +( + cd infra \ + && docker compose build \ + && docker compose run --rm -e DOCS_FORMAT=$DOCS_FORMAT sphinx +) diff --git a/bin/clean.sh b/bin/clean.sh new file mode 100755 index 0000000..e43ad4d --- /dev/null +++ b/bin/clean.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -ex + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# check if _build/ and doctrees exist +if [ ! -d "${SCRIPT_DIR}/../_build/" ]; then + echo "Directory _build/ does not exist, nothing to clean" +else + rm -rf "${SCRIPT_DIR}/../_build/" +fi + +if [ ! -d "${SCRIPT_DIR}/../doctrees/" ]; then + echo "Directory doctrees/ does not exist, nothing to clean" +else + rm -rf "${SCRIPT_DIR}/../doctrees/" +fi + +echo "Cleaned up build artifacts." diff --git a/bin/html.sh b/bin/html.sh new file mode 100755 index 0000000..5959c41 --- /dev/null +++ b/bin/html.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -ex + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +ci_file="$script_dir/ci.html" + +DOCS_FORMAT=html "${SCRIPT_DIR}/ci.sh" diff --git a/build/build.sh b/build/build.sh index d5053e7..20839fa 100755 --- a/build/build.sh +++ b/build/build.sh @@ -4,37 +4,15 @@ ### ~/build $ ./build.sh echo "==== BUILDING readme.pdf ====" +DOCS=$(ls docs/*.md) docker run --rm -it -v $PWD/..:/source jagregory/pandoc --toc -V documentclass=report -s \ README.md \ - hardware-software.md \ - docs/aws.md \ - docs/aws_marketplace.md \ - docs/azure.md \ - docs/bridge.md \ - docs/browser.md \ - docs/configure-investigation.md \ - docs/configure-ontology.md \ - docs/configure-pygraphistry.md \ - docs/configure.md \ - docs/debug-container-networking.md \ - docs/debug-faq.md \ - docs/debug-logs.md \ - docs/deployment-planning.md \ - docs/developer.md \ - docs/manual.md \ - docs/nvidia-docker-in-docker.md \ - docs/rhel_7_6_setup.md \ - docs/templates.md \ - docs/testing-an-install.md \ - docs/threatmodel.md \ - docs/update-backup-migrate.md \ - docs/user-creation.md \ + $DOCS \ -o readme.pdf - echo "==== BUILDING hardware-software.pdf ====" docker run --rm -it -v $PWD/..:/source jagregory/pandoc --toc -V documentclass=report -s \ - hardware-software.md \ + docs/hardware-software.md \ -o hardware-software.pdf ls -al ../*.pdf diff --git a/docs/admin.rst b/docs/admin.rst new file mode 100644 index 0000000..4546e00 --- /dev/null +++ b/docs/admin.rst @@ -0,0 +1,21 @@ +.. toctree:: + :maxdepth: 2 + :hidden: + + README + + +.. toctree:: + :maxdepth: 2 + + commands + planning/index + install/cloud/index + install/on-prem/index + install/testing-an-install + app-config/index + debugging/index + security/index + tools/index + faq + support diff --git a/docs/configure-custom-pivots.md b/docs/app-config/configure-custom-pivots.md similarity index 97% rename from docs/configure-custom-pivots.md rename to docs/app-config/configure-custom-pivots.md index 6fb70d4..94b33c9 100644 --- a/docs/configure-custom-pivots.md +++ b/docs/app-config/configure-custom-pivots.md @@ -86,7 +86,7 @@ One of the most common use cases is creating derived pivots that simplify more g For example, consider expanding based on IP. The base Splunk expand pivot `expand-splunk-plain` could be simplified via: -``` +```json { "id": "expand-splunk-ip", "name": "Splunk: Expand by IP", @@ -136,7 +136,7 @@ Macro variables are of two types: Example: -``` +```json { "id": "expand-splunk-ip", "name": "Splunk: Expand by IP", @@ -144,10 +144,10 @@ Macro variables are of two types: "parameters": [ { "name": "filterPost", - "defaultValue": "head { *max } + "defaultValue": "head { *max }" }, - ... - + ] +} ``` @@ -157,7 +157,7 @@ Macro variables are of two types: Expands to the current value of the corresponding named configuration parameter, or throws an error if no such parameter or it is marked as sensitive -``` +```json { "id": "expand-splunk-searchhead", "name": "Splunk: Expand on searchhead", @@ -167,7 +167,8 @@ Macro variables are of two types: "name": "filter", "defaultValue": "search host={ .splunk.host }" }, - ... + ] +} ``` @@ -175,7 +176,7 @@ Macro variables are of two types: You can add new UI parameters as well. For example, you can create a new pivot that is like `expand-splunk-plain` except also has new text parameter `myNewParam`: -``` +```json { "id": "expand-splunk-custom-param", "name": "Splunk: Expand on searchhead", @@ -188,7 +189,8 @@ You can add new UI parameters as well. For example, you can create a new pivot t "isVisible": true, "defaultValue": "hello", }, - ... + ] +} ``` ## Example: Splunk - Combining new parameters with macros @@ -198,7 +200,7 @@ The power of new parameters comes through macros. For example, an IP search pivo File `config/pivot-db/config/config.json`: -``` +```json { "systemTemplates": { "pivots": [ @@ -220,10 +222,14 @@ File `config/pivot-db/config/config.json`: "defaultValue": "src_ip={ *ip }" }, { - "name": filterPost", + "name": "filterPost", "isVisible": false }, - ... + ] + } + ] + } +} ``` This pivot removes the need for users to know Splunk queries when doing IP searches! @@ -233,7 +239,7 @@ This pivot removes the need for users to know Splunk queries when doing IP searc The following example reuses the Neo4j `search-neo4j` (Cypher query) pivot, where instead of forcing users to write raw Cypher queries for a common domain name lookup, they can just use a new `Domain` text entry button. Note the creation of a new input (`domain`) and the underlying `query` is set to `"isVisible": "false"` and uses the macro `{ *domain }`. -``` +```json { "systemTemplates": { "pivots": [ @@ -255,10 +261,12 @@ The following example reuses the Neo4j `search-neo4j` (Cypher query) pivot, wher "defaultValue": "MATCH (a)-[r:DOMAIN { domain: \"{ *domain }\" }]-(b) RETURN a, r, b" }, { - name: "max", - defaultValue: 2000 + "name": "max", + "defaultValue": 2000 } ] }, - ... + ] + } +} ``` diff --git a/docs/configure-dashboards.md b/docs/app-config/configure-dashboards.md similarity index 100% rename from docs/configure-dashboards.md rename to docs/app-config/configure-dashboards.md diff --git a/docs/configure-investigation.md b/docs/app-config/configure-investigation.md similarity index 99% rename from docs/configure-investigation.md rename to docs/app-config/configure-investigation.md index 26cc0a7..3da9468 100644 --- a/docs/configure-investigation.md +++ b/docs/app-config/configure-investigation.md @@ -41,7 +41,7 @@ After setting these, restart your server: # Schema -The schema specifies how to configure each setting as either an environment variable in `data/config/custom.env` (ex: `"env": "PIVOT_INTERNAL_IP_ACCEPTLIST"`) or as JSON in `data/pivot-db/config/config.json` following the schema's hierarchy + type `"format"` (ex: `{"layouts": {"network": { "ipInternalAcceptList": [ "ip1", "ip2"] } } +The schema specifies how to configure each setting as either an environment variable in `data/config/custom.env` (ex: `"env": "PIVOT_INTERNAL_IP_ACCEPTLIST"`) or as JSON in `data/pivot-db/config/config.json` following the schema's hierarchy + type `"format"` (ex: `{"layouts": {"network": { "ipInternalAcceptList": [ "ip1", "ip2"] } }}`) See also: [main ontology docs](configure-ontology.md) and the [convict specification format](https://github.com/mozilla/node-convict/tree/master/packages/convict) diff --git a/docs/configure-ontology.md b/docs/app-config/configure-ontology.md similarity index 98% rename from docs/configure-ontology.md rename to docs/app-config/configure-ontology.md index b218e1b..c171d20 100644 --- a/docs/configure-ontology.md +++ b/docs/app-config/configure-ontology.md @@ -55,22 +55,19 @@ For example, to create a new node type `ip`, 1. Extend `data/investigations/config/config.json`: -``` +```json { - ... "ontology": { "icons": { "ip": "device", - ..., }, "colors": { "ip": "#F00", - ... }, "sizes": { "ip": 100 } - ... + } } ``` @@ -90,9 +87,8 @@ Graphistry picks the displayed title for each node and edge through the first ma Ex: -``` +```json { - ... "ontology": { "titles": { "byType": { @@ -114,9 +110,8 @@ For example, to recognize `src_ip` and `dest_ip` columns as both generating `ip` 1. Extend `data/investigations/config/config.json`: -``` +```json { - ... "ontology": { "products": [ { @@ -141,7 +136,7 @@ The current set of built-in types is below. Upon system start, Graphistry emits We recommend using built-in types when possible. Each type comes with a built-in color, icon, size, and mappings from common data sources to it. This saves you work now, and as more connectors become supported and new features are added, you will automatically benefit from them in the future as well. -``` +```json [ "actor", "agent", "alert", "amazon", "amex", "arn", "asn", "availabilityzone", "baidu", "bucket", @@ -177,11 +172,11 @@ Layouts have additional options. The most common to modify is to flag values for You can put any regular expression here: -``` +```json "layouts": { "network": { - "ipInternalAcceptList": ['/10\.*/', '/127.0.0.1/'] + "ipInternalAcceptList": ["/10\.*/", "/127.0.0.1/"] } }, diff --git a/docs/configure-pygraphistry.md b/docs/app-config/configure-pygraphistry.md similarity index 98% rename from docs/configure-pygraphistry.md rename to docs/app-config/configure-pygraphistry.md index b2df673..5d3f94b 100644 --- a/docs/configure-pygraphistry.md +++ b/docs/app-config/configure-pygraphistry.md @@ -46,8 +46,9 @@ Deprecated 1.0 API option (api=1, api=2) Override and add query parameters to the loaded visualization iframe by adding key/value strings to `url_params`. This does not control the protocol, domain, nor path, so is primarily for styling and debugging purposes. -##### Example: -``` +#### Example + +```python my_graph.settings(url_params={ 'play': '0', 'my_correlation_id': 'session-123' @@ -67,9 +68,9 @@ register(api=3, username='...', password='...', server=None, protocol=None, api= See PyGraphistry docs for individual connectors such as `.bolt(...)` and `.tigergraph(...)`. -##### Example: Neo4j (bolt/cypher) +#### Example: Neo4j (bolt/cypher) -``` +```python import graphistry graphistry.register(api=3, username='...', password='...', protocol='http', server='my.server.com') @@ -83,6 +84,7 @@ g.cypher("MATCH (a)-[b]->(c) RETURN a,b,c LIMIT 100000").plot() ### Environment variables #### General + | Graphistry Setting | Environment Variable | Description | |:---|:---|:---| | `api_version` | `GRAPHISTRY_API_VERSION` || @@ -114,7 +116,7 @@ Specify a `json` file using key/values from the Settings table. PyGraphistry automatically checks for `graphistry.config` as follows: -``` +```python config_paths = [ os.path.join('/etc/graphistry', '.pygraphistry'), os.path.join(os.path.expanduser('~'), '.pygraphistry'), @@ -130,7 +132,7 @@ Libraries can be added using `pip` in the built-in Jupyter server. To enable new ### Speed up some uploads -``` +```python import graphistry graphistry.register(api=3, username='..', password='...') ``` @@ -138,7 +140,8 @@ graphistry.register(api=3, username='..', password='...') ### Preset a 1.0 API key for all system users Create Python-readable `/etc/graphistry/.pygraphistry`: -``` + +```json { "api_key": "SHARED_USERS_KEY", "protocol": "https", @@ -152,7 +155,7 @@ For Jupyter notebooks, you may want to create per-user login `.pygraphistry` fil In scenarios like Graphistry running on a notebook server, you may prefer to send uploads to a local host (`http://localhost`, `http://nginx`, ...), and tell browser client viewers to use a separate, public host and subpath (`http://graphistry.site.ngo/graphistry`): -``` +```python import os import graphistry diff --git a/docs/configure.md b/docs/app-config/configure.md similarity index 93% rename from docs/configure.md rename to docs/app-config/configure.md index e966857..bb8583a 100644 --- a/docs/configure.md +++ b/docs/app-config/configure.md @@ -2,15 +2,15 @@ Administrators can add users, specify passwords, TLS/SSL, persist data across sessions, connect to databases, specify ontologies, and more. -* For a list of many investigation-oriented options, see their [settings reference page](configure-investigation.md). +* For a list of many investigation-oriented options, see their [settings reference page](configure-investigation). -* See [update, backup, & migration instructions](update-backup-migrate.md) for preserving configurations and data across installations. +* See [update, backup, & migration instructions](../tools/update-backup-migrate.md) for preserving configurations and data across installations. * For advanced Python notebook and application configuration, see [PyGraphistry configuration](configure-pygraphistry.md). ## Add your team and provide API keys -See [user creation docs](user-creation.md) +See [user creation docs](../tools/user-creation.md) ## Top configuration places: data/config/custom.env, data/pivot-db/config/config.json @@ -41,7 +41,7 @@ Recommendations for SSO when self-hosting: ## TLS -We encourage everyone to use HTTPS over HTTP, especially through the automatic TLS option, for [securing authentication](authentication.md) +We encourage everyone to use HTTPS over HTTP, especially through the automatic TLS option, for [securing authentication](../security/authentication.md) Graphistry supports both free automatic TLS within your server (Caddy/LetsEncrypt) and offloading to an external load balancer or proxy @@ -50,6 +50,7 @@ As part of TLS configuration, decide whether to allow cross-origin embedding of ### TLS Hardening #### Caddy + If Caddy is used for TLS, several additional policies may be of interest, but only use ones that match your intended usage patterns: ``` @@ -102,6 +103,7 @@ COOKIE_SAMESITE=None Caddy supports [free automatic TLS](https://caddyserver.com/docs/automatic-https) as long as your site meets the listed conditions. Sample `Caddyfile`: + ``` https://*.website.org:443 { tls { @@ -119,7 +121,7 @@ Note that Graphistry is internally configured via `nginx`. ### Setup free Semi-Automatic TLS -The above step may fail if your server does not allow outbound internet traffic. To work around, perform a [manual LetsEncrypt Certbot handshake for Caddy](configure-tls-caddy-manual-lets-encrypt-handshake.md). +The above step may fail if your server does not allow outbound internet traffic. To work around, perform a [manual LetsEncrypt Certbot handshake for Caddy](../security/configure-tls-caddy-manual-lets-encrypt-handshake.md). ### Setup custom certs @@ -214,7 +216,7 @@ You may externally redirect traffic, such as through an AWS Load Balancer or Clo If `80` / `443` are already taken, such as when running multiple Graphistry instances on the same box, you may want to change to another port. For example, to expose public port `8888` for Graphistry's HTTP traffic instead of `80`, configure `docker-compose.yml` to map it as follows: -```yml +```yaml caddy: ... expose: @@ -236,7 +238,7 @@ Separately [configure the public and private dashboards](configure-dashboards.md ## Connectors -Optionally, you can configure Graphistry to use database connectors. Graphistry will orchestrate cross-database query generation, pushing them down through the database API, and returning the combined results to the user. This means Graphistry can reuse your existing scaleout infrastructure and make its data more accessible to your users without requiring a second copy to be maintained. Some connectors further support use of the [Graphistry data bridge](bridge.md) for proxying requests between a Graphistry cloud server and an intermediate on-prem data bridge instead of directly connecting to on-prem API servers. +Optionally, you can configure Graphistry to use database connectors. Graphistry will orchestrate cross-database query generation, pushing them down through the database API, and returning the combined results to the user. This means Graphistry can reuse your existing scaleout infrastructure and make its data more accessible to your users without requiring a second copy to be maintained. Some connectors further support use of the [Graphistry data bridge](../tools/bridge.md) for proxying requests between a Graphistry cloud server and an intermediate on-prem data bridge instead of directly connecting to on-prem API servers. ### Security Notes @@ -349,11 +351,11 @@ For faster start, or to be explicit about schema, or avoid start issues, you may ### Data bridge -In scenarios such as a Graphistry cloud server accessing on-prem API servers, an intermediate on-prem data bridge may be necessary. You can mix bridged and direct (default) connectors in the same Graphistry server. Learn more about the [Graphistry data bridge](bridge.md). +In scenarios such as a Graphistry cloud server accessing on-prem API servers, an intermediate on-prem data bridge may be necessary. You can mix bridged and direct (default) connectors in the same Graphistry server. Learn more about the [Graphistry data bridge](../tools/bridge.md). ### Variants -* Give your Graphistry implementation user increased permissions so they can embed Graphistry into existing dashbboard and notification systems, such as for embedded visualizations and quicklinks into contextual [investigation templates](templates.md)* Run a [Graphistry data bridge](bridge.md), if available for your connector, which may help with cases such as firewalls preventing incoming connections from Graphistry to your database +* Give your Graphistry implementation user increased permissions so they can embed Graphistry into existing dashbboard and notification systems, such as for embedded visualizations and quicklinks into contextual [investigation templates](templates.md)* Run a [Graphistry data bridge](../tools/bridge.md), if available for your connector, which may help with cases such as firewalls preventing incoming connections from Graphistry to your database * Run a bastion server between Graphistry and your database, such as a new Splunk search head * Create fine-grained permissions by running multiple Graphistry virtual servers, with a new Splunk role per instance @@ -372,5 +374,5 @@ Every connector comes with a base set of pivots. See [custom pivots](configure-c ## Performance -See [performance tuning](performance-tuning.md) +See [performance tuning](../debugging/performance-tuning.md) diff --git a/docs/connector-management.md b/docs/app-config/connector-management.md similarity index 99% rename from docs/connector-management.md rename to docs/app-config/connector-management.md index 99a4448..954297e 100644 --- a/docs/connector-management.md +++ b/docs/app-config/connector-management.md @@ -203,7 +203,7 @@ ACTION=update_all_pats CONNECTOR_ID="sample_connector_id" PATS_CSV='path/to/pats #### CSV File Example -```csv +``` username,PAT user123,abc123pat user456,def456pat diff --git a/docs/email.md b/docs/app-config/email.md similarity index 100% rename from docs/email.md rename to docs/app-config/email.md diff --git a/docs/app-config/index.rst b/docs/app-config/index.rst new file mode 100644 index 0000000..4a5a612 --- /dev/null +++ b/docs/app-config/index.rst @@ -0,0 +1,37 @@ + +App configuration +================= + + +Visual Playbooks +---------------- + +.. toctree:: + :maxdepth: 1 + :titlesonly: + + configure + configure-custom-pivots + configure-investigation + configure-ontology + connector-management + templates + +Streamlit +--------- + +.. toctree:: + :maxdepth: 1 + :titlesonly: + + configure-dashboards + +System +------ + +.. toctree:: + :maxdepth: 1 + :titlesonly: + + email + configure-pygraphistry diff --git a/docs/templates.md b/docs/app-config/templates.md similarity index 97% rename from docs/templates.md rename to docs/app-config/templates.md index 777e283..31ca2cd 100644 --- a/docs/templates.md +++ b/docs/app-config/templates.md @@ -82,22 +82,22 @@ To make a template appear as a Workflow Action on a specific kind of event: ## 6. Best practices -#### Manual data for first step +### Manual data for first step By making the first step an `Enter data` one, most of the parameters can be set on it. The URL generates an initial graph, and subsequent steps expand on them. -#### Multiple entry points +### Multiple entry points You can likely combine multiple templates into one. For example, in IT scenarios, 360 views for IP's, MAC addresses, and host names likely look the same. Make the first step create a graph for one or more of these, the next ones derive one value type from the other (or a canonical ID), and the remaining steps look the same. -#### Set time range and provide instructions +### Set time range and provide instructions Analysts unfamiliar with your template would strongly benefit from instructions telling them what to modify (if anything) and how to use the investigation. Many options likely have sane defaults on a per-template basis, such as the time range, so we recommend including them in your URLs. -#### Naming +### Naming Content management can become an issue. Use a custom short description name, such as `name=%5BPhone%20360%5D%20555-5555` (=> `[Phone 360] 555-5555`. The generated investigations can now be easily searched and sorted. -#### Cross-linking +### Cross-linking You can include templates as links within templates! For example, whenever a phone number node is generated, you can include attribute `link` with value `/pivot/template?investigation=...` . diff --git a/docs/commands.md b/docs/commands.md new file mode 100644 index 0000000..5be3509 --- /dev/null +++ b/docs/commands.md @@ -0,0 +1,34 @@ +# Top commands + +Graphistry supports advanced command-line administration via standard `docker-compose`, `.yml` / `.env` files, and `caddy` reverse-proxy configuration. + +## Login to server + +| Image | Command | +|--: |:-- | +| **AWS** | `ssh -i [your_key].pem ubuntu@[your_public_ip]` | +| **Azure** | `ssh -i [your_key].pem [your_user]@[your_public_ip]`
`ssh [your_user]@[your_public_ip]` (pwd-based) | +| **Google** | `gcloud compute [your_instance] ssh` | +| **On-prem / BYOL** | Contact your admin | + +## CLI commands + +All likely require `sudo`. Run from where your `docker-compose.yml` file is located: `/home/ubuntu/graphistry` (AWS), `/var/graphistry` (Azure), or `/var/graphistry//` (recommended on-prem). + +| TASK | COMMAND | NOTES | +|--: |:--- |:--- | +| **Install** | `docker load -i containers.tar.gz` | Install the `containers.tar.gz` Graphistry release from the current folder. You may need to first run `tar -xvvf my-graphistry-release.tar.gz`. | +| **Start
interactive** | `docker-compose up` | Starts Graphistry, close with ctrl-c | +| **Start
daemon** | `docker-compose up -d` | Starts Graphistry as background process | +| **Start
namespaced (concurrent)** | `docker-compose -p my_unique_namespace up` | Starts Graphistry in a specific namespace. Enables running multiple independent instances of Graphistry. NOTE: Must modify Caddy service in `docker-compose.yml` to use non-conflicting public ports, and likewise change global volumes to be independent. | +| **Stop** | `docker-compose stop` | Stops Graphistry | +| **Restart (soft)** | `docker restart ` | Soft restart. May also need to restart service `nginx`. | +| **Restart (hard)** | `docker up -d --force-recreate --no-deps ` | Restart with fresh state. May also need to restart service `nginx`. | +| **Reset** | `docker-compose down -v && docker-compose up -d` | Stop Graphistry, remove all internal state (including the user account database!), and start fresh . | +| **Status** | `docker-compose ps`, `docker ps`, and `docker status` | Status: Uptime, healthchecks, ... | +| **GPU Status** | `nvidia-smi` | See GPU processes, compute/memory consumption, and driver. Ex: `watch -n 1.5 nvidia-smi`. Also, `docker run --rm -it nvidia/cuda:11.5.0-base-ubuntu20.04 nvidia-smi` for in-container test. | +| **1.0 API Key** | docker-compose exec streamgl-vgraph-etl curl "http://0.0.0.0:8080/api/internal/provision?text=MYUSERNAME" | Generates API key for a developer or notebook user (1.0 API is deprecated)| +| **Logs** | `docker-compose logs ` | Ex: Watch all logs, starting with the 20 most recent lines: `docker-compose logs -f -t --tail=20 forge-etl-python` . You likely need to switch Docker to use the local json logging driver by deleting the two default managed Splunk log driver options in `/etc/docker/daemon.json` and then restarting the `docker` daemon (see below). | +| **Create Users** | Use Admin Panel (see [Create Users](tools/user-creation.md)) or `etc/scripts/rest` | +| **Restart Docker Daemon** | `sudo service docker restart` | Use when changing `/etc/docker/daemon.json`, ... | +| **Jupyter shell**| `docker exec -it -u root graphistry_notebook_1 bash` then `source activate rapids` | Use for admin tasks like global package installs | \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..6e94394 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,625 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import docutils.nodes, os, logging, re, sys +from docutils import nodes +from docutils.parsers.rst import roles +from packaging.version import Version +from sphinx.application import Sphinx +from sphinx.transforms.post_transforms import SphinxTransform + + +sys.path.insert(0, os.path.abspath("../..")) +import graphistry + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +# -- Project information ----------------------------------------------------- + +project = "Graphistry CLI" +copyright = "2024, Graphistry, Inc." +author = "Graphistry, Inc." + +html_title = "Graphistry Administration Documentation" +html_short_title = "Graphistry Admin" +html_logo = 'static/graphistry_banner_transparent_colored.png' +html_favicon = 'static/favicon.ico' + +# The full version, including alpha/beta/rc tags +version = str(Version(graphistry.__version__)) +release = version + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'myst_parser', + 'nbsphinx', + "sphinx.ext.autodoc", + #'sphinx.ext.autosummary', + 'sphinx.ext.intersphinx', + "sphinx.ext.ifconfig", + #"sphinx_autodoc_typehints", + "sphinx_copybutton", +] + +myst_url_schemes = ["http", "https", "mailto", "ftp", "file"] +myst_heading_anchors = 3 # Allow md files to have links with extension .md in the path + +#myst_enable_extensions = ["linkify", "substitution"] +myst_enable_extensions = ["linkify"] +myst_commonmark_only = False # Use CommonMark only +myst_heading_anchors_auto_id_prefix = False # Disable auto ID prefix for headings + +# TODO guarantee most notebooks are executable (=> maintained) +# and switch to opt'ing out the few that are hard, e.g., DB deps +nbsphinx_execute = 'never' +nbsphinx_allow_errors = False # Allow errors in notebooks + +autodoc_typehints = "description" +always_document_param_types = True +typehints_document_rtype = True + +#suppress_warnings = [ +# 'nbsphinx.localfile', # Suppresses local file warnings in notebooks +#] + +nitpick_ignore = [] + +#set_type_checking_flag = True + +# typehints_fully_qualified=True +always_document_param_types = True +typehints_document_rtype = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +# source_suffix = ['.rst', '.ipynb'] +source_suffix = { + '.md': 'myst', + '.txt': 'markdown', + '.rst': 'restructuredtext', + #'.ipynb': 'nbsphinx', +} +# The master toctree document. +root_doc = "index" + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [ + + '_build', + '**/_build/**', + 'doctrees', + '**/doctrees/**', + '**/*.txt', + +] + +pygments_style = "sphinx" +todo_include_todos = False + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +#html_theme = "sphinx_rtd_theme" +html_theme = "sphinx_book_theme" + + +html_theme_options = { + "repository_url": "https://github.com/graphistry/graphistry-cli", + "use_repository_button": True, + + # Optional top horizontal navigation bar + #"navbar_start": ["navbar-start.html"], + #"navbar_center": ["navbar-center.html"], + #"navbar_end": ["navbar-end.html"], + + "logo": { + #"link": "https://www.graphistry.com/get-started", + #"text": "Graphistry, Inc.", + "alt_text": "Graphistry, Inc." + } +} + + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['static'] # '_static' +# html_css_files = ['graphistry.css'] + +html_show_sphinx = False + +htmlhelp_basename = "Graphistry-admin-doc" + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + 'preamble': r''' + + \usepackage{graphicx} + % Command to adjust image paths for LaTeX builds + \makeatletter + \def\input@path{{./}{./_static/img/}{./static/img/}} + \makeatother + + \usepackage{svg} % Enables SVG handling via Inkscape + + \RequirePackage{etex} % Ensure extended TeX capacities + \usepackage[utf8]{inputenc} % Enable UTF-8 support + \usepackage[T1]{fontenc} % Use T1 font encoding for better character support + \usepackage{lmodern} % Load Latin Modern fonts for better quality + \usepackage{amsmath} % For advanced math formatting + \usepackage{amsfonts} % For math fonts + \usepackage{amssymb} % For additional math symbols + \usepackage{graphicx} % For including images + \usepackage{hyperref} % For hyperlinks + \usepackage{textcomp} % For additional text symbols + \usepackage{breakurl} % Allows line breaks in URLs + \usepackage{listings} % For code listings + \usepackage{float} % Improved control of floating objects + \usepackage{microtype} % Improves text appearance with microtypography + \usepackage{lipsum} % For generating dummy text (if needed) + + + % Increase capacity limits + \setcounter{totalnumber}{200} % Maximum floats + \setcounter{dbltopnumber}{200} % Double float maximum + \setcounter{secnumdepth}{3} % Section numbering depth + \setcounter{tocdepth}{3} % Table of contents depth + + % Increase dimensions and allocations + \usepackage{morefloats} % Allows for more floats + \setlength{\emergencystretch}{3em} % Help with overfull hboxes + \setlength{\maxdepth}{100pt} % Sets a high limit for max depth (if applicable) + + % Allocate more memory for TeX + \usepackage{etex} % Use eTeX for more memory + %\reserveinserts{200} % Reserve more inserts + \setcounter{totalnumber}{200} % Ensure maximum floats are increased + + + % Declare Unicode characters + \DeclareUnicodeCharacter{1F389}{\textbf{(party popper)}} + \DeclareUnicodeCharacter{1F3C6}{\textbf{(trophy)}} + \DeclareUnicodeCharacter{1F44D}{\textbf{(thumbs up)}} + \DeclareUnicodeCharacter{1F4AA}{\textbf{Strong}} % Muscle emoji + \DeclareUnicodeCharacter{1F4B0}{\textbf{Money Bag}} % Money bag emoji (text representation) + \DeclareUnicodeCharacter{1F525}{\textbf{(fire)}} + \DeclareUnicodeCharacter{1F600}{\textbf{(grinning)}} + \DeclareUnicodeCharacter{1F609}{\textbf{(winking)}} + \DeclareUnicodeCharacter{1F614}{\textbf{(pensive)}} + \DeclareUnicodeCharacter{1F680}{\textbf{(rocket)}} + \DeclareUnicodeCharacter{2501}{\textbf{━}} % Heavy horizontal line + \DeclareUnicodeCharacter{2588}{\textbf{█}} % Full block character + \DeclareUnicodeCharacter{258A}{\textbf{▊}} % Center right block character + \DeclareUnicodeCharacter{258B}{\textbf{▉}} % Right block character + \DeclareUnicodeCharacter{258C}{\textbf{▌}} % Center block character + \DeclareUnicodeCharacter{258D}{\textbf{▍}} % Center left block character + \DeclareUnicodeCharacter{258E}{\textbf{▎}} % Left third block character + \DeclareUnicodeCharacter{258F}{\textbf{▏}} % Right block character + \DeclareUnicodeCharacter{2728}{\textbf{(sparkles)}} + \DeclareUnicodeCharacter{2764}{\textbf{(heart)}} + \DeclareUnicodeCharacter{2705}{\textbf{(check mark)}} + \DeclareUnicodeCharacter{2B50}{\textbf{(star)}} + + ''', +} + +# Use pdflatex as the LaTeX engine +latex_engine = 'pdflatex' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ( + root_doc, + "Graphistry.tex", + u"Graphistry Admin Documentation", + u"Graphistry, Inc.", + "manual", + ), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +latex_domain_indices = False + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(root_doc, "graphistry-admin-docs", u"Graphistry Admin Documentation", [author], 1)] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + root_doc, + "Graphistry", + u"Praphistry Admin Documentation", + author, + "Graphistry", + "Admin documents for Graphistry.", + "Miscellaneous", + ), +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +texinfo_domain_indices = False + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# texinfo_no_detailmenu = False + + +# -- Options for Epub output ---------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project +epub_author = author +epub_publisher = author +epub_copyright = copyright + +# The basename for the epub file. It defaults to the project name. +# epub_basename = project + +# The HTML theme for the epub output. Since the default themes are not optimized +# for small screen space, using the same theme for HTML and epub output is +# usually not wise. This defaults to 'epub', a theme designed to save visual +# space. +# epub_theme = 'epub' + +# The language of the text. It defaults to the language option +# or 'en' if the language is not set. +# epub_language = '' + +# The scheme of the identifier. Typical schemes are ISBN or URL. +# epub_scheme = '' + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# epub_identifier = '' + +# A unique identification for the text. +# epub_uid = '' + +# A tuple containing the cover image and cover page html template filenames. +# epub_cover = () + +# A sequence of (type, uri, title) tuples for the guide element of content.opf. +# epub_guide = () + +# HTML files that should be inserted before the pages created by sphinx. +# The format is a list of tuples containing the path and title. +# epub_pre_files = [] + +# HTML files shat should be inserted after the pages created by sphinx. +# The format is a list of tuples containing the path and title. +# epub_post_files = [] + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ["search.html"] + +# The depth of the table of contents in toc.ncx. +# epub_tocdepth = 3 + +# Allow duplicate toc entries. +# epub_tocdup = True + +# Choose between 'default' and 'includehidden'. +# epub_tocscope = 'default' + +# Fix unsupported image types using the Pillow. +# epub_fix_images = False + +# Scale large images. +# epub_max_image_width = 0 + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# epub_show_urls = 'inline' + +# If false, no index is generated. +# epub_use_index = True + + +# Example configuration for intersphinx: refer to the Python standard library. +# intersphinx_mapping = {'https://docs.python.org/': None} + + + +# -- Custom Preprocessor Configuration --------------------------------------- + +def replace_iframe_src(app, doctree, docname): + """ + Replace relative iframe src paths with absolute URLs in HTML content. + Specifically targets iframe tags with src attributes starting with /graph/. + """ + # Define a flexible regex pattern to match