feat: Add script to convert .dot file to graph database

This commit adds a new Python script `convert.py` that converts a `.dot` file into a graph database. The script uses the `pydot` library to parse the `.dot` file and creates a graph object. It then parses the labels of nodes using regular expressions and updates the traversal code accordingly.

The script also loads environment variables using `dotenv`, establishes a connection with Neo4j using the provided URI, username, and password, and uploads the parsed nodes and relationships to the graph database.

The commit also includes the addition of a sample `.dot` file in `src/stargripcorp.dataplatform.infra.azure/graph.dot`.

scripts/dot_to_graphdb/convert.py

@@ -0,0 +1,95 @@
+import pydot
+import os
+import re
+from neo4j import GraphDatabase
+from dotenv import load_dotenv
+
+# Load the environment variables
+load_dotenv()
+
+# Load the .dot file and create a graph
+dot_file_path = 'src/stargripcorp.dataplatform.infra.azure/graph.dot'
+graphs = pydot.graph_from_dot_file(dot_file_path)
+if graphs is None:
+    raise ValueError('Invalid .dot file')
+
+graph = graphs[0]
+
+# Function to parse the labels
+def parse_label(label):
+    label = label.replace('"', '')  # Remove quotes from the label
+    pattern = re.compile(
+        r'^urn:pulumi:(?P<stack>[^:]+)::[^:]+::pkg:(?P<pkg>[^:]+):(?P<type>[^$]+)\$(?P<subtype>[^:]+):(?P<name>.+)$'
+    )
+    match = pattern.match(label)
+    if match:
+        return match.groupdict()
+    return None  # Return None if regex match fails
+
+# Updating the traversal code to parse the labels
+nodes, relationships = {}, []
+for edge in graph.get_edges():
+    relationships.append((edge.get_source(), edge.get_destination()))
+for node in graph.get_nodes():
+    node_name = node.get_name()
+    label = node.get_label() if node.get_label() else node_name  # Default to node name if label is absent
+    parsed_label = parse_label(label)
+    if parsed_label:
+        nodes[node_name] = parsed_label
+    else:
+        nodes[node_name] = {'name': node_name}  # Default to node name if parsing fails
+
+# URI examples: 'neo4j://localhost', 'neo4j+s://xxx.databases.neo4j.io'
+URI = os.getenv('NEO4J_URI')
+user_name = os.getenv('NEO4J_USER')
+password = os.getenv('NEO4J_PASSWORD')
+AUTH =(user_name,password)
+
+if URI is None:
+    raise ValueError('Invalid URI')
+
+
+
+def upload_graph(tx, nodes, relationships):
+    # Create nodes, with parsed label information
+    for node_name, label_info in nodes.items():
+
+        n_name=label_info.get('name', node_name)
+        n_type=label_info.get('type', 'Unknown')
+        n_stack=label_info.get('stack','Unknown')
+        n_pkg=label_info.get('pkg', 'Unknown')
+        n_subtype=label_info.get('subtype', 'Unknown')
+
+        if n_name != node_name:
+            tx.run(
+            """
+                MERGE (:PulumiNode {
+                    id: $id,
+                    name: $name, 
+                    type: $type, 
+                    stack: $stack,
+                    pkg: $pkg,
+                    subtype: $subtype
+                })
+                """, 
+                id=node_name,  # node_name serves as the id
+                name=n_name,
+                type=n_type,
+                stack=n_stack,
+                pkg=n_pkg,
+                subtype=n_subtype
+            )
+    # Create relationships
+    for source, destination in relationships:
+        tx.run(
+            '''
+            MATCH (a:PulumiNode {id: $source}), (b:PulumiNode {id: $destination})
+            MERGE (a)-[:DEPENDS_ON]->(b)
+            ''',
+            source=source, destination=destination
+        )
+
+# Invoke the script
+with GraphDatabase.driver(URI, auth=AUTH) as driver:
+    with driver.session() as session:
+        session.execute_write(upload_graph, nodes, relationships)

src/stargripcorp.dataplatform.infra.azure/graph.dot

@@ -0,0 +1,51 @@
+strict digraph {
+    Resource0 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault$azure-native:authorization:RoleAssignment::merca-data-dev-ra-3a668e53-336b-4d30-94bf-6620cdd036ec"];
+    Resource0 -> Resource1 [color = "#AA6639"];
+    Resource2 [label="urn:pulumi:current::stargripcorp::pulumi:pulumi:Stack::stargripcorp-current"];
+    Resource3 [label="urn:pulumi:current::stargripcorp::pkg:azure:resource_group$azure-native:consumption:Budget::merca-core-dev-budget"];
+    Resource3 -> Resource4 [color = "#AA6639"];
+    Resource5 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault$azure-native:keyvault:Secret::merca-data-dev-secret"];
+    Resource5 -> Resource1 [color = "#AA6639"];
+    Resource6 [label="urn:pulumi:current::stargripcorp::pkg:azure:resource_group$azure-native:resources:ResourceGroup::merca-data-dev-rg"];
+    Resource6 -> Resource7 [color = "#AA6639"];
+    Resource6 -> Resource8 [color = "#246C60", label = "resourceGroupName"];
+    Resource6 -> Resource9 [color = "#246C60", label = "resourceGroupName"];
+    Resource6 -> Resource5 [color = "#246C60", label = "resourceGroupName"];
+    Resource1 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault::data-kv"];
+    Resource1 -> Resource2 [color = "#AA6639"];
+    Resource10 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault::core-kv"];
+    Resource10 -> Resource2 [color = "#AA6639"];
+    Resource11 [label="urn:pulumi:current::stargripcorp::pkg:azure:resource_group$azure-native:resources:ResourceGroup::merca-core-dev-rg"];
+    Resource11 -> Resource4 [color = "#AA6639"];
+    Resource11 -> Resource12 [color = "#246C60", label = "resourceGroupName"];
+    Resource11 -> Resource13 [color = "#246C60", label = "resourceGroupName"];
+    Resource14 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault$azure-native:authorization:RoleAssignment::merca-core-dev-ra-3a668e53-336b-4d30-94bf-6620cdd036ec"];
+    Resource14 -> Resource10 [color = "#AA6639"];
+    Resource15 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault$azure-native:authorization:RoleAssignment::merca-core-dev-ra-f5d889ea-e64e-467a-9240-f875ff284c04"];
+    Resource15 -> Resource10 [color = "#AA6639"];
+    Resource7 [label="urn:pulumi:current::stargripcorp::pkg:azure:resource_group::data-rg"];
+    Resource7 -> Resource2 [color = "#AA6639"];
+    Resource16 [label="urn:pulumi:current::stargripcorp::pkg:azure:resource_group$azure-native:consumption:Budget::merca-data-dev-budget"];
+    Resource16 -> Resource7 [color = "#AA6639"];
+    Resource8 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault$azure-native:keyvault:Vault::merca-data-dev-kv"];
+    Resource8 -> Resource1 [color = "#AA6639"];
+    Resource8 -> Resource17 [color = "#246C60", label = "scope"];
+    Resource8 -> Resource5 [color = "#246C60", label = "vaultName"];
+    Resource8 -> Resource0 [color = "#246C60", label = "scope"];
+    Resource17 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault$azure-native:authorization:RoleAssignment::merca-data-dev-ra-f5d889ea-e64e-467a-9240-f875ff284c04"];
+    Resource17 -> Resource1 [color = "#AA6639"];
+    Resource4 [label="urn:pulumi:current::stargripcorp::pkg:azure:resource_group::core-rg"];
+    Resource4 -> Resource2 [color = "#AA6639"];
+    Resource18 [label="urn:pulumi:current::stargripcorp::pulumi:providers:azure-native::default_2_9_0"];
+    Resource12 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault$azure-native:keyvault:Vault::merca-core-dev-kv"];
+    Resource12 -> Resource14 [color = "#246C60", label = "scope"];
+    Resource12 -> Resource10 [color = "#AA6639"];
+    Resource12 -> Resource15 [color = "#246C60", label = "scope"];
+    Resource12 -> Resource13 [color = "#246C60", label = "vaultName"];
+    Resource13 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault$azure-native:keyvault:Secret::merca-core-dev-secret"];
+    Resource13 -> Resource10 [color = "#AA6639"];
+    Resource19 [label="urn:pulumi:current::stargripcorp::pkg:azure:storage::data-sa"];
+    Resource19 -> Resource2 [color = "#AA6639"];
+    Resource9 [label="urn:pulumi:current::stargripcorp::pkg:azure:storage$azure-native:storage:StorageAccount::merca-data-dev-sa"];
+    Resource9 -> Resource19 [color = "#AA6639"];
+}