Merge pull request #119 from ArpitShukla12/PR_staging_integration_sup…

…port

Added Github and Gitlab support

.github/workflows/test-action.yml

@@ -2,7 +2,7 @@ name: Test Action
 
 on:
   pull_request:
-    types: [ opened, edited, synchronize, reopened, closed ]
+    types: [opened, edited, synchronize, reopened, closed]
 
 jobs:
   get-downstream-assets:
@@ -28,4 +28,4 @@ jobs:
             main: DBT-DEMO-PROD
             beta: Wide World Importers PE1
             test-action: Wide World Importers PE1
-          IGNORE_MODEL_ALIAS_MATCHING: true
+          IGNORE_MODEL_ALIAS_MATCHING: true

adapters/api/create-resource.js

@@ -0,0 +1,64 @@
+import { v4 as uuidv4 } from "uuid";
+import fetch from "node-fetch";
+import stringify from "json-stringify-safe";
+import {
+  ATLAN_INSTANCE_URL,
+  ATLAN_API_TOKEN,
+} from "../utils/get-environment-variables.js";
+
+export default async function createResource(
+  guid,
+  name,
+  link,
+  sendSegmentEventOfIntegration
+) {
+  var myHeaders = {
+    Authorization: `Bearer ${ATLAN_API_TOKEN}`,
+    "Content-Type": "application/json",
+  };
+
+  var raw = stringify({
+    entities: [
+      {
+        typeName: "Link",
+        attributes: {
+          qualifiedName: uuidv4(),
+          name,
+          link,
+          tenantId: "default",
+        },
+        relationshipAttributes: {
+          asset: {
+            guid,
+          },
+        },
+      },
+    ],
+  });
+
+  var requestOptions = {
+    method: "POST",
+    headers: myHeaders,
+    body: raw,
+  };
+
+  var response = await fetch(
+    `${ATLAN_INSTANCE_URL}/api/meta/entity/bulk`,
+    requestOptions
+  )
+    .then((e) => e.json())
+    .catch((err) => {
+      console.log(err);
+      sendSegmentEventOfIntegration({
+        action: "dbt_ci_action_failure",
+        properties: {
+          reason: "failed_to_create_resource",
+          asset_name: name, // This should change
+          msg: err,
+        },
+      });
+    });
+
+  if (response?.errorCode) return null;
+  return response;
+}

adapters/api/get-asset.js

@@ -0,0 +1,149 @@
+import fetch from "node-fetch";
+import stringify from "json-stringify-safe";
+import {
+  getErrorModelNotFound,
+  getErrorDoesNotMaterialize,
+} from "../templates/atlan.js";
+import {
+  ATLAN_INSTANCE_URL,
+  ATLAN_API_TOKEN,
+} from "../utils/get-environment-variables.js";
+
+export default async function getAsset({
+  name,
+  sendSegmentEventOfIntegration,
+  environment,
+  integration,
+}) {
+  var myHeaders = {
+    Authorization: `Bearer ${ATLAN_API_TOKEN}`,
+    "Content-Type": "application/json",
+  };
+
+  var raw = stringify({
+    dsl: {
+      from: 0,
+      size: 21,
+      query: {
+        bool: {
+          must: [
+            {
+              match: {
+                __state: "ACTIVE",
+              },
+            },
+            {
+              match: {
+                "__typeName.keyword": "DbtModel",
+              },
+            },
+            {
+              match: {
+                "name.keyword": name,
+              },
+            },
+            ...(environment
+              ? [
+                  {
+                    term: {
+                      "assetDbtEnvironmentName.keyword": environment,
+                    },
+                  },
+                ]
+              : []),
+          ],
+        },
+      },
+    },
+    attributes: [
+      "name",
+      "description",
+      "userDescription",
+      "sourceURL",
+      "qualifiedName",
+      "connectorName",
+      "certificateStatus",
+      "certificateUpdatedBy",
+      "certificateUpdatedAt",
+      "ownerUsers",
+      "ownerGroups",
+      "classificationNames",
+      "meanings",
+      "dbtModelSqlAssets",
+    ],
+    relationAttributes: [
+      "name",
+      "description",
+      "assetDbtProjectName",
+      "assetDbtEnvironmentName",
+      "connectorName",
+      "certificateStatus",
+    ],
+  });
+
+  var requestOptions = {
+    method: "POST",
+    headers: myHeaders,
+    body: raw,
+  };
+
+  var response = await fetch(
+    `${ATLAN_INSTANCE_URL}/api/meta/search/indexsearch#findAssetByExactName`,
+    requestOptions
+  )
+    .then((e) => e.json())
+    .catch((err) => {
+      sendSegmentEventOfIntegration({
+        action: "dbt_ci_action_failure",
+        properties: {
+          reason: "failed_to_get_asset",
+          asset_name: name,
+          msg: err,
+        },
+      });
+    });
+
+  if (!response?.entities?.length) {
+    return {
+      error: getErrorModelNotFound(name),
+    };
+  }
+
+  if (Array.isArray(response.entities)) {
+    response.entities.sort((entityA, entityB) => {
+      const hasDbtModelSqlAssetsA =
+        entityA.attributes.dbtModelSqlAssets &&
+        entityA.attributes.dbtModelSqlAssets.length > 0;
+      const hasDbtModelSqlAssetsB =
+        entityB.attributes.dbtModelSqlAssets &&
+        entityB.attributes.dbtModelSqlAssets.length > 0;
+
+      if (hasDbtModelSqlAssetsA && !hasDbtModelSqlAssetsB) {
+        return -1; // entityA comes before entityB
+      } else if (!hasDbtModelSqlAssetsA && hasDbtModelSqlAssetsB) {
+        return 1; // entityB comes before entityA
+      }
+
+      // Primary sorting criterion: Latest createTime comes first
+      if (entityA.createTime > entityB.createTime) {
+        return -1;
+      } else if (entityA.createTime < entityB.createTime) {
+        return 1;
+      }
+
+      return 0; // No difference in sorting for these two entities
+    });
+  }
+
+  if (!response?.entities[0]?.attributes?.dbtModelSqlAssets?.length > 0)
+    return {
+      error: getErrorDoesNotMaterialize(
+        name,
+        ATLAN_INSTANCE_URL,
+        response,
+        integration
+      ),
+    };
+
+  return response.entities[0];
+}

adapters/api/get-classifications.js

@@ -0,0 +1,37 @@
+import fetch from "node-fetch";
+import {
+  ATLAN_INSTANCE_URL,
+  ATLAN_API_TOKEN,
+} from "../utils/get-environment-variables.js";
+
+export default async function getClassifications({
+  sendSegmentEventOfIntegration,
+}) {
+  var myHeaders = {
+    Authorization: `Bearer ${ATLAN_API_TOKEN}`,
+    "Content-Type": "application/json",
+  };
+
+  var requestOptions = {
+    method: "GET",
+    headers: myHeaders,
+    redirect: "follow",
+  };
+
+  var response = await fetch(
+    `${ATLAN_INSTANCE_URL}/api/meta/types/typedefs?type=classification`,
+    requestOptions
+  )
+    .then((e) => e.json())
+    .catch((err) => {
+      sendSegmentEventOfIntegration({
+        action: "dbt_ci_action_failure",
+        properties: {
+          reason: "failed_to_get_classifications",
+          msg: err,
+        },
+      });
+    });
+
+  return response?.classificationDefs;
+}

adapters/api/get-downstream-assets.js

@@ -0,0 +1,127 @@
+import fetch from "node-fetch";
+import {
+  getConnectorImage,
+  getCertificationImage,
+  getImageURL,
+} from "../utils/index.js";
+import stringify from "json-stringify-safe";
+import {
+  ATLAN_INSTANCE_URL,
+  ATLAN_API_TOKEN,
+} from "../utils/get-environment-variables.js";
+
+const ASSETS_LIMIT = 100;
+
+export default async function getDownstreamAssets(
+  asset,
+  guid,
+  totalModifiedFiles,
+  sendSegmentEventOfIntegration,
+  integration
+) {
+  var myHeaders = {
+    authorization: `Bearer ${ATLAN_API_TOKEN}`,
+    "content-type": "application/json",
+  };
+
+  var raw = stringify({
+    guid: guid,
+    size: Math.max(Math.ceil(ASSETS_LIMIT / totalModifiedFiles), 1),
+    from: 0,
+    depth: 21,
+    direction: "OUTPUT",
+    entityFilters: {
+      condition: "AND",
+      criterion: [
+        {
+          attributeName: "__typeName",
+          operator: "not_contains",
+          attributeValue: "Process",
+        },
+        {
+          attributeName: "__state",
+          operator: "eq",
+          attributeValue: "ACTIVE",
+        },
+      ],
+    },
+    attributes: [
+      "name",
+      "description",
+      "userDescription",
+      "sourceURL",
+      "qualifiedName",
+      "connectorName",
+      "certificateStatus",
+      "certificateUpdatedBy",
+      "certificateUpdatedAt",
+      "ownerUsers",
+      "ownerGroups",
+      "classificationNames",
+      "meanings",
+    ],
+    excludeMeanings: false,
+    excludeClassifications: false,
+  });
+
+  var requestOptions = {
+    method: "POST",
+    headers: myHeaders,
+    body: raw,
+  };
+
+  var handleError = (err) => {
+    const comment = `### ${getConnectorImage(
+      asset.attributes.connectorName
+    )} [${asset.displayText}](${ATLAN_INSTANCE_URL}/assets/${
+      asset.guid
+    }/overview?utm_source=dbt_${integration}_action) ${
+      asset.attributes?.certificateStatus
+        ? getCertificationImage(asset.attributes.certificateStatus)
+        : ""
+    }
+            
+_Failed to fetch impacted assets._
+            
+${getImageURL(
+  "atlan-logo",
+  15,
+  15
+)} [View lineage in Atlan](${ATLAN_INSTANCE_URL}/assets/${
+      asset.guid
+    }/lineage/overview?utm_source=dbt_${integration}_action)`;
+
+    sendSegmentEventOfIntegration({
+      action: "dbt_ci_action_failure",
+      properties: {
+        reason: "failed_to_fetch_lineage",
+        asset_guid: asset.guid,
+        asset_name: asset.name,
+        asset_typeName: asset.typeName,
+        msg: err,
+      },
+    });
+
+    return comment;
+  };
+
+  var response = await fetch(
+    `${ATLAN_INSTANCE_URL}/api/meta/lineage/list`,
+    requestOptions
+  )
+    .then((e) => {
+      if (e.status === 200) {
+        return e.json();
+      } else {
+        throw e;
+      }
+    })
+    .catch((err) => {
+      return {
+        error: handleError(err),
+      };
+    });
+  if (response.error) return response;
+
+  return response;
+}