From 4afffee4521ebbe36bd4a995402d9d61f7fa85ab Mon Sep 17 00:00:00 2001 From: Valentyn Kahamlyk Date: Fri, 1 Mar 2024 22:15:47 -0800 Subject: [PATCH] Added compatibility with Neptune Analytics (#241) * Added server timeout config + fetch timeout * updated README * env to configure proxy server timeout and retry * use the new variable names * removed RequestSig class and use IAMService environment variable in node-server.js. * Add Neptune graph permissions and update IAM service type * revert and adjust * optinal serviceType * Fix service type casing in environment variables and headers * Remove unnecessary console.log statement in node-server.js * pass service-type to proxy * pickup SERVICE_TYPE from env * code review suggestions * Update packages/graph-explorer/src/utils/constants.ts Co-authored-by: Alexey Temnikov * fixed schema fetch for Neptune Analytics. Cleanup * ... --------- Co-authored-by: Juan Cubeddu Co-authored-by: Juan Cubeddu Co-authored-by: Valentyn Kahamlyk Co-authored-by: Alexey Temnikov --- .gitignore | 2 +- Changelog.md | 4 ++ README.md | 4 ++ additionaldocs/ecs/ECS_FARGATE_DEPLOYMENT.md | 5 ++ .../sagemaker/graph-explorer-policy.json | 7 ++ .../sagemaker/install-graph-explorer-lc.sh | 1 + .../graph-explorer-proxy-server/RequestSig.js | 27 ------- .../node-server.js | 38 ++++++---- packages/graph-explorer/.env | 3 + .../openCypher/queries/fetchSchema.ts | 1 + .../src/connector/openCypher/useOpenCypher.ts | 2 +- .../src/connector/useGEFetch.ts | 2 + .../src/core/ConfigurationProvider/types.ts | 4 ++ packages/graph-explorer/src/index.tsx | 2 + .../ConnectionDetail/ConnectionDetail.tsx | 3 +- .../CreateConnection/CreateConnection.tsx | 72 +++++++++++++------ .../graph-explorer/src/utils/constants.ts | 1 + packages/graph-explorer/src/utils/index.ts | 1 + process-environment.sh | 1 + 19 files changed, 114 insertions(+), 66 deletions(-) delete mode 100644 packages/graph-explorer-proxy-server/RequestSig.js create mode 100644 packages/graph-explorer/src/utils/constants.ts diff --git a/.gitignore b/.gitignore index 2c93ee560..ae219abef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ **/node_modules/ **/coverage/ **/.DS_Store +**/.vs/ **/.idea/ -**/.vs/ \ No newline at end of file diff --git a/Changelog.md b/Changelog.md index 3cad479da..8fc482d95 100644 --- a/Changelog.md +++ b/Changelog.md @@ -13,6 +13,10 @@ This release includes the following feature enhancements and bug fixes: - Bumped `vite` to `4.5.2` () - Searching for text containing quotes now works. +**Features** + +- Added compatibility with Neptune Analitycs. + ## Release 1.5.0 This release includes the following feature enhancements and bug fixes: diff --git a/README.md b/README.md index 86a2cf2b6..db1a05b4f 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ You can create and manage connections to graph databases using this feature. Con - **Using proxy server:** Check this box if using a proxy endpoint. - **Graph connection URL:** Provide the endpoint for the graph database - **AWS IAM Auth Enabled:** Check this box if connecting to Amazon Neptune using IAM Auth and SigV4 signed requests + - **Service Type:** Choose the service type - **AWS Region:** Specify the AWS region where the Neptune cluster is hosted (e.g., us-east-1) - **Fetch Timeout:** Specify the timeout for the fetch request @@ -122,6 +123,7 @@ To provide a default connection such that initial loads of the graph explorer al - `GRAPH_CONNECTION_URL` - `None` - See [Add a New Connection](#connections-ui) - Required if `USING_PROXY_SERVER=True` and `IAM=True` - `AWS_REGION` - `None` - See [Add a New Connection](#connections-ui) + - `SERVICE_TYPE` - `neptune-db`, Set this as `neptune-db` for Neptune database or `neptune-graph` for Neptune Analytics. #### JSON Configuration Approach @@ -133,6 +135,7 @@ First, create a `config.json` file containing values for the connection attribut "GRAPH_CONNECTION_URL": "https://cluster-cqmizgqgrsbf.us-west-2.neptune.amazonaws.com:8182", "USING_PROXY_SERVER": true, (Can be string or boolean) "IAM": true, (Can be string or boolean) + "SERVICE_TYPE": "neptune-db", "AWS_REGION": "us-west-2", "GRAPH_TYPE": "gremlin" (Possible Values: "gremlin", "sparql", "opencypher"), "GRAPH_EXP_HTTPS_CONNECTION": true (Can be string or boolean), @@ -160,6 +163,7 @@ docker run -p 80:80 -p 443:443 \ --env IAM=false \ --env GRAPH_CONNECTION_URL=https://cluster-cqmizgqgrsbf.us-west-2.neptune.amazonaws.com:8182 \ --env AWS_REGION=us-west-2 \ + --env SERVICE_TYPE=neptune-db \ --env PROXY_SERVER_HTTPS_CONNECTION=true \ --env GRAPH_EXP_FETCH_REQUEST_TIMEOUT=240000 \ graph-explorer diff --git a/additionaldocs/ecs/ECS_FARGATE_DEPLOYMENT.md b/additionaldocs/ecs/ECS_FARGATE_DEPLOYMENT.md index eecc1b4db..f6229bb47 100644 --- a/additionaldocs/ecs/ECS_FARGATE_DEPLOYMENT.md +++ b/additionaldocs/ecs/ECS_FARGATE_DEPLOYMENT.md @@ -113,6 +113,10 @@ After the request is processed, the console will return you to your certificate "name": "HOST", "value": "localhost" }, + { + "name": "SERVICE_TYPE", + "value": "neptune-db" + }, { "name": "GRAPH_CONNECTION_URL", "value": "https://{NEPTUNE_ENDPOINT}:8182" @@ -157,6 +161,7 @@ After the request is processed, the console will return you to your certificate - `IAM`: Set this to `true` to use SigV4 signed requests, if your Neptune cluster has IAM db authentication enabled. - `GRAPH_CONNECTION_URL`: Set this as `https://{NEPTUNE_ENDPOINT}:8182`. - `PUBLIC_OR_PROXY_ENDPOINT`: Set this as `https://{Domain name set in Step 5 of "Request an ACM Public Certificate"}`. + - `SERVICE_TYPE`: Set this as `neptune-db` for Neptune database or `neptune-graph` for Neptune Analytics. 6. Click **Create**. ### Create a Fargate Service diff --git a/additionaldocs/sagemaker/graph-explorer-policy.json b/additionaldocs/sagemaker/graph-explorer-policy.json index d5f21b2d8..a95d7b080 100644 --- a/additionaldocs/sagemaker/graph-explorer-policy.json +++ b/additionaldocs/sagemaker/graph-explorer-policy.json @@ -8,6 +8,13 @@ "arn:aws:neptune-db:[AWS_REGION]:[AWS_ACCOUNT_ID]:[NEPTUNE_CLUSTER_RESOURCE_ID]/*" ] }, + { + "Effect": "Allow", + "Action": "neptune-graph:*", + "Resource": [ + "arn:aws:neptune-graph:[AWS_REGION]:[AWS_ACCOUNT_ID]:[NEPTUNE_CLUSTER_RESOURCE_ID]/*" + ] + }, { "Effect": "Allow", "Action": "sagemaker:DescribeNotebookInstance", diff --git a/additionaldocs/sagemaker/install-graph-explorer-lc.sh b/additionaldocs/sagemaker/install-graph-explorer-lc.sh index 2687a6b55..14d368818 100644 --- a/additionaldocs/sagemaker/install-graph-explorer-lc.sh +++ b/additionaldocs/sagemaker/install-graph-explorer-lc.sh @@ -4,6 +4,7 @@ sudo -u ec2-user -i <<'EOF' echo "export GRAPH_NOTEBOOK_AUTH_MODE=DEFAULT" >> ~/.bashrc # set to IAM instead of DEFAULT if cluster is IAM enabled echo "export GRAPH_NOTEBOOK_HOST=CHANGE-ME" >> ~/.bashrc +echo "export GRAPH_NOTEBOOK_SERVICE=neptune-db" >> ~/.bashrc # set to `neptune-db` for Neptune database or `neptune-graph` for Neptune Analytics echo "export GRAPH_NOTEBOOK_PORT=8182" >> ~/.bashrc echo "export AWS_REGION=us-west-2" >> ~/.bashrc # modify region if needed diff --git a/packages/graph-explorer-proxy-server/RequestSig.js b/packages/graph-explorer-proxy-server/RequestSig.js deleted file mode 100644 index 38c2ddddc..000000000 --- a/packages/graph-explorer-proxy-server/RequestSig.js +++ /dev/null @@ -1,27 +0,0 @@ -const aws4 = require("aws4"); - -class RequestSig { - _algorithm = "AWS4-HMAC-SHA256"; - _region; - _service = "neptune-db"; - _host; - _ac; - _sac; - - constructor(host, region, accessKey, secretAccessKey, sessionToken) { - this._region = region; - this._host = host; - this._ac = accessKey; - this._sac = secretAccessKey; - this._st = sessionToken; - } - - requestAuthHeaders(inputPort, requestedPath) { - var opts = { host: this._host.split(":")[0], path: requestedPath, service: this._service, region: this._region, port: inputPort }; - return aws4.sign(opts, {accessKeyId: this._ac, secretAccessKey: this._sac, sessionToken: this._st}); - } - } - - module.exports = { - RequestSig - }; \ No newline at end of file diff --git a/packages/graph-explorer-proxy-server/node-server.js b/packages/graph-explorer-proxy-server/node-server.js index abec9e304..6adbf4cb1 100644 --- a/packages/graph-explorer-proxy-server/node-server.js +++ b/packages/graph-explorer-proxy-server/node-server.js @@ -16,6 +16,9 @@ const aws4 = require("aws4"); // Load environment variables from .env file. dotenv.config({ path: "../graph-explorer/.env" }); +const DEFAULT_SERVICE_TYPE = "neptune-db"; +const NEPTUNE_ANALYTICS_SERVICE_TYPE = "neptune-graph"; + // Create a logger instance with pino. const proxyLogger = pino({ level: process.env.LOG_LEVEL || "info", @@ -69,6 +72,7 @@ const retryFetch = async ( options, isIamEnabled, region, + serviceType, retryDelay = 10000, refetchMaxRetries = 1 ) => { @@ -78,7 +82,7 @@ const retryFetch = async ( host: url.hostname, port: url.port, path: url.pathname + url.search, - service: "neptune-db", + service: serviceType, region, method: options.method, body: options.body ?? undefined, @@ -88,7 +92,7 @@ const retryFetch = async ( host: url.hostname, port: url.port, path: url.pathname + url.search, - service: "neptune-db", + service: serviceType, region, method: options.method, body: options.body ?? undefined, @@ -99,7 +103,7 @@ const retryFetch = async ( host: url.hostname, port: url.port, path: url.pathname + url.search, - service: "neptune-db", + service: serviceType, method: options.method, body: options.body ?? undefined, headers: options.headers, @@ -129,13 +133,14 @@ const retryFetch = async ( }; // Function to fetch data from the given URL and send it as a response. -async function fetchData(res, next, url, options, isIamEnabled, region) { +async function fetchData(res, next, url, options, isIamEnabled, region, serviceType) { try { const response = await retryFetch( new URL(url), options, isIamEnabled, - region + region, + serviceType ); const data = await response.json(); res.send(data); @@ -185,8 +190,9 @@ async function fetchData(res, next, url, options, isIamEnabled, region) { }; const isIamEnabled = !!req.headers["aws-neptune-region"]; const region = isIamEnabled ? req.headers["aws-neptune-region"] : ""; + const serviceType = isIamEnabled ? (req.headers["service-type"] ?? DEFAULT_SERVICE_TYPE) : ""; - fetchData(res, next, rawUrl, requestOptions, isIamEnabled, region); + fetchData(res, next, rawUrl, requestOptions, isIamEnabled, region, serviceType); }); // POST endpoint for Gremlin queries. @@ -209,8 +215,9 @@ async function fetchData(res, next, url, options, isIamEnabled, region) { const isIamEnabled = !!req.headers["aws-neptune-region"]; const region = isIamEnabled ? req.headers["aws-neptune-region"] : ""; + const serviceType = isIamEnabled ? (req.headers["service-type"] ?? DEFAULT_SERVICE_TYPE) : ""; - fetchData(res, next, rawUrl, requestOptions, isIamEnabled, region); + fetchData(res, next, rawUrl, requestOptions, isIamEnabled, region, serviceType); }); // POST endpoint for openCypher queries. @@ -233,22 +240,26 @@ async function fetchData(res, next, url, options, isIamEnabled, region) { const isIamEnabled = !!req.headers["aws-neptune-region"]; const region = isIamEnabled ? req.headers["aws-neptune-region"] : ""; + const serviceType = isIamEnabled ? (req.headers["service-type"] ?? DEFAULT_SERVICE_TYPE) : ""; - fetchData(res, next, rawUrl, requestOptions, isIamEnabled, region); + fetchData(res, next, rawUrl, requestOptions, isIamEnabled, region, serviceType); }); - // GET endpoint to retrieve PostgreSQL statistics summary. + // GET endpoint to retrieve statistics summary. app.get("/pg/statistics/summary", async (req, res, next) => { - const rawUrl = `${req.headers["graph-db-connection-url"]}/pg/statistics/summary?mode=detailed`; + const isIamEnabled = !!req.headers["aws-neptune-region"]; + const serviceType = isIamEnabled ? (req.headers["service-type"] ?? DEFAULT_SERVICE_TYPE) : ""; + const rawUrl = serviceType === NEPTUNE_ANALYTICS_SERVICE_TYPE + ? `${req.headers["graph-db-connection-url"]}/summary?mode=detailed` + : `${req.headers["graph-db-connection-url"]}/pg/statistics/summary?mode=detailed`; const requestOptions = { method: "GET", }; - const isIamEnabled = !!req.headers["aws-neptune-region"]; const region = isIamEnabled ? req.headers["aws-neptune-region"] : ""; - fetchData(res, next, rawUrl, requestOptions, isIamEnabled, region); + fetchData(res, next, rawUrl, requestOptions, isIamEnabled, region, serviceType); }); // GET endpoint to retrieve RDF statistics summary. @@ -261,8 +272,9 @@ async function fetchData(res, next, url, options, isIamEnabled, region) { const isIamEnabled = !!req.headers["aws-neptune-region"]; const region = isIamEnabled ? req.headers["aws-neptune-region"] : ""; + const serviceType = isIamEnabled ? (req.headers["service-type"] ?? DEFAULT_SERVICE_TYPE) : ""; - fetchData(res, next, rawUrl, requestOptions, isIamEnabled, region); + fetchData(res, next, rawUrl, requestOptions, isIamEnabled, region, serviceType); }); app.get("/logger", (req, res, next) => { diff --git a/packages/graph-explorer/.env b/packages/graph-explorer/.env index d394b7654..890c83a14 100644 --- a/packages/graph-explorer/.env +++ b/packages/graph-explorer/.env @@ -2,3 +2,6 @@ LOG_LEVEL=info ## Client side fetch request timeout GRAPH_EXP_FETCH_REQUEST_TIMEOUT=240000 +## Service +SERVICE_TYPE=neptune-db + diff --git a/packages/graph-explorer/src/connector/openCypher/queries/fetchSchema.ts b/packages/graph-explorer/src/connector/openCypher/queries/fetchSchema.ts index 98772eefd..1b2cb7393 100644 --- a/packages/graph-explorer/src/connector/openCypher/queries/fetchSchema.ts +++ b/packages/graph-explorer/src/connector/openCypher/queries/fetchSchema.ts @@ -79,6 +79,7 @@ const fetchVerticesAttributes = async ( const response = await openCypherFetch(verticesTemplate); const vertex = response.results[0]?.object as OCVertex; + if (!vertex) return; const label = vertex["~labels"][0] as string; const properties = vertex["~properties"]; vertices.push({ diff --git a/packages/graph-explorer/src/connector/openCypher/useOpenCypher.ts b/packages/graph-explorer/src/connector/openCypher/useOpenCypher.ts index 20b98ede0..4ab75c438 100644 --- a/packages/graph-explorer/src/connector/openCypher/useOpenCypher.ts +++ b/packages/graph-explorer/src/connector/openCypher/useOpenCypher.ts @@ -37,7 +37,7 @@ const useOpenCypher = () => { ...ops }); - summary = response.payload.graphSummary as GraphSummary || undefined; + summary = (response.payload ? response.payload.graphSummary as GraphSummary : response.graphSummary as GraphSummary) || undefined; } catch (e) { if (import.meta.env.DEV) { console.error("[Summary API]", e); diff --git a/packages/graph-explorer/src/connector/useGEFetch.ts b/packages/graph-explorer/src/connector/useGEFetch.ts index 98d86f5c9..51924d02a 100644 --- a/packages/graph-explorer/src/connector/useGEFetch.ts +++ b/packages/graph-explorer/src/connector/useGEFetch.ts @@ -2,6 +2,7 @@ import { useCallback } from 'react'; import localforage from "localforage"; import { CacheItem } from './useGEFetchTypes'; import { useConfiguration, type ConnectionConfig } from '../core'; +import { DEFAULT_SERVICE_TYPE } from "../utils/constants"; // 10 minutes const CACHE_TIME_MS = 10 * 60 * 1000; @@ -46,6 +47,7 @@ const useGEFetch = () => { } if (connection?.awsAuthEnabled) { headers["aws-neptune-region"] = connection.awsRegion || ""; + headers["service-type"] = connection.serviceType || DEFAULT_SERVICE_TYPE; } return { ...headers, ...typeHeaders }; diff --git a/packages/graph-explorer/src/core/ConfigurationProvider/types.ts b/packages/graph-explorer/src/core/ConfigurationProvider/types.ts index ecfa22cf8..124f204b7 100644 --- a/packages/graph-explorer/src/core/ConfigurationProvider/types.ts +++ b/packages/graph-explorer/src/core/ConfigurationProvider/types.ts @@ -131,6 +131,10 @@ export type ConnectionConfig = { * If it is Neptune, it could need authentication. */ awsAuthEnabled?: boolean; + /** + * If it is Neptune, it could need authentication. + */ + serviceType?: 'neptune-db' | 'neptune-graph', /** * AWS Region where the Neptune cluster is deployed. * It is needed to sign requests. diff --git a/packages/graph-explorer/src/index.tsx b/packages/graph-explorer/src/index.tsx index 2ce70e31c..e00de0bb7 100644 --- a/packages/graph-explorer/src/index.tsx +++ b/packages/graph-explorer/src/index.tsx @@ -6,6 +6,7 @@ import App from "./App"; import { RawConfiguration } from "./core"; import ConnectedProvider from "./core/ConnectedProvider"; import "./index.css"; +import { DEFAULT_SERVICE_TYPE } from "./utils/constants"; const grabConfig = async (): Promise => { const defaultConnectionPath = `${location.origin}/defaultConnection`; @@ -62,6 +63,7 @@ const grabConfig = async (): Promise => { graphDbUrl: defaultConnectionData.GRAPH_EXP_CONNECTION_URL || "", awsAuthEnabled: !!defaultConnectionData.GRAPH_EXP_IAM, awsRegion: defaultConnectionData.GRAPH_EXP_AWS_REGION || "", + serviceType: defaultConnectionData.SERVICE_TYPE || DEFAULT_SERVICE_TYPE, fetchTimeoutMs: defaultConnectionData.GRAPH_EXP_FETCH_REQUEST_TIMEOUT || 240000, }, diff --git a/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionDetail.tsx b/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionDetail.tsx index 7f602fe55..251f53bf6 100644 --- a/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionDetail.tsx +++ b/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionDetail.tsx @@ -210,13 +210,14 @@ const ConnectionDetail = ({ isSync, onSyncChange }: ConnectionDetailProps) => { setEdit(false)} configId={config.id} - disabledFields={config.__fileBase ? ["type", "url"] : undefined} + disabledFields={config.__fileBase ? ["type", "url", "serviceType"] : undefined} initialData={{ ...(config.connection || {}), name: config.displayLabel || config.id, url: config.connection?.url, type: config.connection?.queryEngine, fetchTimeMs: config.connection?.fetchTimeoutMs, + serviceType: config.connection?.serviceType, }} /> diff --git a/packages/graph-explorer/src/modules/CreateConnection/CreateConnection.tsx b/packages/graph-explorer/src/modules/CreateConnection/CreateConnection.tsx index d3ef6cef6..aa23ea8d1 100644 --- a/packages/graph-explorer/src/modules/CreateConnection/CreateConnection.tsx +++ b/packages/graph-explorer/src/modules/CreateConnection/CreateConnection.tsx @@ -29,6 +29,7 @@ type ConnectionForm = { proxyConnection?: boolean; graphDbUrl?: string; awsAuthEnabled?: boolean; + serviceType?: "neptune-db" | "neptune-graph"; awsRegion?: string; enableCache?: boolean; cacheTimeMs?: number; @@ -39,15 +40,15 @@ export const CONNECTIONS_OP: { label: string; value: NonNullable; }[] = [ - { label: "PG (Property Graph) - Gremlin", value: "gremlin" }, - { label: "PG (Property Graph) - OpenCypher", value: "openCypher" }, - { label: "RDF (Resource Description Framework) - SPARQL", value: "sparql" }, + { label: "Gremlin - PG (Property Graph)", value: "gremlin" }, + { label: "OpenCypher - PG (Property Graph)", value: "openCypher" }, + { label: "SPARQL - RDF (Resource Description Framework)", value: "sparql" }, ]; export type CreateConnectionProps = { configId?: string; initialData?: ConnectionForm; - disabledFields?: Array<"name" | "type" | "url">; + disabledFields?: Array<"name" | "type" | "url" | "serviceType">; onClose(): void; }; @@ -74,6 +75,7 @@ const CreateConnection = ({ proxyConnection: data.proxyConnection, graphDbUrl: data.graphDbUrl, awsAuthEnabled: data.awsAuthEnabled, + serviceType: data.serviceType, awsRegion: data.awsRegion, enableCache: data.enableCache, cacheTimeMs: data.cacheTimeMs * 60 * 1000, @@ -103,6 +105,7 @@ const CreateConnection = ({ proxyConnection: data.proxyConnection, graphDbUrl: data.graphDbUrl, awsAuthEnabled: data.awsAuthEnabled, + serviceType: data.serviceType, awsRegion: data.awsRegion, cacheTimeMs: data.cacheTimeMs * 60 * 1000, fetchTimeoutMs: data.fetchTimeMs, @@ -144,6 +147,7 @@ const CreateConnection = ({ proxyConnection: initialData?.proxyConnection || false, graphDbUrl: initialData?.graphDbUrl || "", awsAuthEnabled: initialData?.awsAuthEnabled || false, + serviceType: initialData?.serviceType || "neptune-db", awsRegion: initialData?.awsRegion || "", enableCache: true, cacheTimeMs: (initialData?.cacheTimeMs ?? 10 * 60 * 1000) / 60000, @@ -153,10 +157,18 @@ const CreateConnection = ({ const [hasError, setError] = useState(false); const onFormChange = useCallback( (attribute: string) => (value: number | string | string[] | boolean) => { - setForm(prev => ({ - ...prev, - [attribute]: value, - })); + if (attribute === "serviceType" && value === "neptune-graph") { + setForm(prev => ({ + ...prev, + [attribute]: value, + ["type"]: "openCypher", + })); + } else { + setForm(prev => ({ + ...prev, + [attribute]: value, + })); + } }, [] ); @@ -173,7 +185,7 @@ const CreateConnection = ({ return; } - if (form.awsAuthEnabled && !form.awsRegion) { + if (form.awsAuthEnabled && (!form.awsRegion || !form.serviceType)) { setError(true); return; } @@ -199,7 +211,7 @@ const CreateConnection = ({ options={CONNECTIONS_OP} value={form.type} onChange={onFormChange("type")} - isDisabled={disabledFields?.includes("type")} + isDisabled={disabledFields?.includes("type") || form.serviceType === "neptune-graph"} />
)} {form.proxyConnection && form.awsAuthEnabled && ( -
- -
+ <> +
+ +
+
+