diff --git a/adapters/gateway.js b/adapters/gateway.js index a933b65..df652c5 100644 --- a/adapters/gateway.js +++ b/adapters/gateway.js @@ -5,5 +5,5 @@ export async function runAction(token, integrationModule) { return; } const integration = new integrationModule(token); - await integration.run(); + await integration.run(); // Add error part } diff --git a/adapters/integrations/github-integration.js b/adapters/integrations/github-integration.js index 45ecf94..adfd11f 100644 --- a/adapters/integrations/github-integration.js +++ b/adapters/integrations/github-integration.js @@ -2,17 +2,22 @@ import dotenv from "dotenv"; // Check do we actually need it or not import IntegrationInterface from "./contract/contract.js"; import github from "@actions/github"; +import { isIgnoreModelAliasMatching } from "../../src/utils/get-environment-variables.js"; +import { getConnectorImage } from "../../src/utils/get-image-url.js"; +import { getEnvironments } from "../../src/utils/get-environment-variables.js"; import { getAsset, getDownstreamAssets, sendSegmentEvent, createResource, + getClassifications, } from "../../src/api/index.js"; -import { - renderDownstreamAssetsComment, - getImageURL, - auth, -} from "../../src/utils/index.js"; +import { getImageURL, auth } from "../../src/utils/index.js"; +import { isDev } from "../../src/utils/get-environment-variables.js"; +import { truncate } from "../../src/utils/create-comment.js"; +import { getInstanceUrl } from "../../src/utils/get-environment-variables.js"; +const IS_DEV = isDev(); +const ATLAN_INSTANCE_URL = getInstanceUrl(); dotenv.config(); @@ -22,6 +27,8 @@ export default class GitHubIntegration extends IntegrationInterface { } async run() { + //Done + //Complete console.log("Run Github"); const timeStart = Date.now(); const { context } = github; @@ -30,6 +37,8 @@ export default class GitHubIntegration extends IntegrationInterface { const { state, merged } = pull_request; if (!(await this.authIntegration({ octokit, context }))) { + //DONE + //Complete throw { message: "Wrong API Token" }; } @@ -43,6 +52,7 @@ export default class GitHubIntegration extends IntegrationInterface { if (total_assets !== 0) { this.sendSegmentEventOfIntegration("dbt_ci_action_run", { + //Complete asset_count: total_assets, total_time: Date.now() - timeStart, }); @@ -50,30 +60,47 @@ export default class GitHubIntegration extends IntegrationInterface { } async printDownstreamAssets({ octokit, context }) { - // Implementation for printing impact on GitHub - // Use this.token to access the token - - const changedFiles = await this.getChangedFiles({ octokit, context }); // Complete - + //Done + const changedFiles = await this.getChangedFiles(octokit, context); //Complete let comments = ``; let totalChangedFiles = 0; - for (const { fileName, filePath } of changedFiles) { - const assetName = await this.getAssetName({ - // Complete + for (const { fileName, filePath, status } of changedFiles) { + const aliasName = await this.getAssetName({ + //Complete octokit, context, fileName, filePath, }); + const assetName = isIgnoreModelAliasMatching() ? fileName : aliasName; //Complete + + const environments = getEnvironments(); + + let environment = null; + for (const [baseBranchName, environmentName] of environments) { + if (baseBranchName === context.payload.pull_request.base.ref) { + environment = environmentName; + break; + } + } + const asset = await getAsset({ - //Complete + //Done name: assetName, sendSegmentEventOfIntegration: this.sendSegmentEventOfIntegration, + environment: environment, + integration: "github", }); - // TODO :- When we call getAsset we are always sending segment event to github. We have to resolve this. - // Either we can pass that function to getAsset function to resolve this. Or check for better alternatives. - // If we pass the function we can simply write getAsset({name: assetName}, this.sendSegmentEvent) + + if (totalChangedFiles !== 0) comments += "\n\n---\n\n"; + + if (status === "added") { + comments += `### ${getConnectorImage("dbt")} ${fileName} 🆕 +Its a new model and not present in Atlan yet, you'll see the downstream impact for it after its present in Atlan.`; + totalChangedFiles++; + continue; + } if (asset.error) { comments += asset.error; @@ -81,18 +108,18 @@ export default class GitHubIntegration extends IntegrationInterface { continue; } - const { guid } = asset.attributes.sqlAsset; + const materialisedAsset = asset.attributes.dbtModelSqlAssets[0]; const timeStart = Date.now(); + const totalModifiedFiles = changedFiles.filter( + (i) => i.status === "modified" + ).length; const downstreamAssets = await getDownstreamAssets( //Complete asset, - guid, - this.sendSegmentEventOfIntegration, - "github" + materialisedAsset.guid, + totalModifiedFiles ); - if (totalChangedFiles !== 0) comments += "\n\n---\n\n"; - if (downstreamAssets.error) { comments += downstreamAssets.error; totalChangedFiles++; @@ -100,13 +127,21 @@ export default class GitHubIntegration extends IntegrationInterface { } this.sendSegmentEventOfIntegration("dbt_ci_action_downstream_unfurl", { + //Complete asset_guid: asset.guid, asset_type: asset.typeName, - downstream_count: downstreamAssets.length, + downstream_count: downstreamAssets.entities.length, total_fetch_time: Date.now() - timeStart, }); - // WHICH CODE TO PICKUP ONE IN THE BETA OR IN THE GITLAB BRANCH + + const classifications = await getClassifications({ + //Complete + sendSegmentEventOfIntegration: this.sendSegmentEventOfIntegration, + }); + const comment = await this.renderDownstreamAssetsComment({ + //Done + //Complete octokit, context, asset, @@ -127,19 +162,20 @@ Here is your downstream impact analysis for **${totalChangedFiles} ${ ${comments}`; - const existingComment = await this.checkCommentExists({ octokit, context }); + const existingComment = await this.checkCommentExists({ octokit, context }); //Complete - if (totalChangedFiles > 0) { + if (totalChangedFiles > 0) await this.createIssueComment({ + //Complete octokit, context, content: comments, comment_id: existingComment?.id, }); - } if (totalChangedFiles === 0 && existingComment) await this.deleteComment({ + //Complete octokit, context, comment_id: existingComment.id, @@ -149,11 +185,9 @@ ${comments}`; } async setResourceOnAsset({ octokit, context }) { - //COMPLETE - // Implementation for setting resources on GitHub - // Use this.token to access the token - - const changedFiles = await this.getChangedFiles({ octokit, context }); + //Done + //Complete + const changedFiles = await this.getChangedFiles({ octokit, context }); //Complete const { pull_request } = context.payload; var totalChangedFiles = 0; @@ -167,33 +201,54 @@ ${comments}`; fileName, filePath, }); + + const environments = getEnvironments(); + + let environment = null; + for (const [baseBranchName, environmentName] of environments) { + if (baseBranchName === context.payload.pull_request.base.ref) { + environment = environmentName; + break; + } + } + const asset = await getAsset({ + //Done name: assetName, sendSegmentEventOfIntegration: this.sendSegmentEventOfIntegration, + environment: environment, + integration: "github", }); - if (!asset) continue; + if (asset.error) continue; const { guid: modelGuid } = asset; - const { guid: tableAssetGuid } = asset.attributes.sqlAsset; + const { guid: tableAssetGuid } = + asset?.attributes?.dbtModelSqlAssets?.[0]; + + if (modelGuid) + await createResource( + //Complete + modelGuid, + "Pull Request on GitHub", + pull_request.html_url, + this.sendSegmentEventOfIntegration + ); - await createResource( - modelGuid, - "Pull Request on GitHub", - pull_request.html_url, - this.sendSegmentEventOfIntegration - ); - await createResource( - tableAssetGuid, - "Pull Request on GitHub", - pull_request.html_url, - this.sendSegmentEventOfIntegration - ); + if (tableAssetGuid) + await createResource( + //Complete + tableAssetGuid, + "Pull Request on GitHub", + pull_request.html_url, + this.sendSegmentEventOfIntegration + ); totalChangedFiles++; } const comment = await this.createIssueComment({ + //Complete octokit, context, content: `🎊 Congrats on the merge! @@ -208,18 +263,15 @@ This pull request has been added as a resource to all the assets modified. ✅ } async authIntegration({ octokit, context }) { + //DONE //COMPLETE - // IMPORT ATLAN INSTANCE - // Implement your auth logic here const response = await auth(); - // Need to change the route we are passing in createIssueComment. - const existingComment = await checkCommentExists(octokit, context); + const existingComment = await this.checkCommentExists({ octokit, context }); console.log("Existing Comment", existingComment); if (response?.status === 401) { await this.createIssueComment( - // Check how u are calling this createIssueComment function octokit, context, `We couldn't connect to your Atlan Instance, please make sure to set the valid Atlan Bearer Token as \`ATLAN_API_TOKEN\` as this repository's action secret. @@ -253,8 +305,8 @@ Set your repository action secrets [here](https://github.com/${context.payload.r } async sendSegmentEventOfIntegration({ action, properties }) { - //COMPLETE - // Implement your sendSegmentEvent logic here + //Done + //FullyComplete // IMPORT ATLAN_INSTANCE_URL. const domain = new URL(ATLAN_INSTANCE_URL).hostname; @@ -274,7 +326,8 @@ Set your repository action secrets [here](https://github.com/${context.payload.r } async getChangedFiles({ octokit, context }) { - //Complete + //Done + //FullyComplete const { repository, pull_request } = context.payload, owner = repository.owner.login, repo = repository.name, @@ -322,10 +375,15 @@ Set your repository action secrets [here](https://github.com/${context.payload.r } async getAssetName({ octokit, context, fileName, filePath }) { - // Complete + //Done + // FullyComplete var regExp = /{{\s*config\s*\(\s*(?:[^,]*,)*\s*alias\s*=\s*['"]([^'"]+)['"](?:\s*,[^,]*)*\s*\)\s*}}/im; - var fileContents = await this.getFileContents(octokit, context, filePath); + var fileContents = await this.getFileContents({ + octokit, + context, + filePath, + }); if (fileContents) { var matches = regExp.exec(fileContents); @@ -339,7 +397,8 @@ Set your repository action secrets [here](https://github.com/${context.payload.r } async getFileContents({ octokit, context, filePath }) { - // Complete + //Done + // FullyComplete const { repository, pull_request } = context.payload, owner = repository.owner.login, repo = repository.name, @@ -367,7 +426,8 @@ Set your repository action secrets [here](https://github.com/${context.payload.r } async checkCommentExists({ octokit, context }) { - //COMPLETE + //Done + //FullyComplete if (IS_DEV) return null; const { pull_request } = context.payload; @@ -387,7 +447,8 @@ Set your repository action secrets [here](https://github.com/${context.payload.r } async createIssueComment({ - // COMPLETE + //Done + // FullyComplete octokit, context, content, @@ -405,7 +466,7 @@ ${content}`; body: content, }; - console.log(content); + console.log(content, content.length); if (IS_DEV) return content; @@ -415,7 +476,8 @@ ${content}`; } async deleteComment({ octokit, context, comment_id }) { - //COMPLETE + //Done + //FullyComplete const { pull_request } = context.payload; return octokit.rest.issues.deleteComment({ @@ -426,6 +488,8 @@ ${content}`; } async renderDownstreamAssetsComment({ + //Done + //FullyComplete octokit, context, asset, diff --git a/adapters/integrations/gitlab-integration.js b/adapters/integrations/gitlab-integration.js index 06a3ba4..35639c8 100644 --- a/adapters/integrations/gitlab-integration.js +++ b/adapters/integrations/gitlab-integration.js @@ -10,8 +10,12 @@ import { sendSegmentEvent, } from "../../src/api/index.js"; import { getImageURL, auth } from "../../src/utils/index.js"; - +import { getGitLabEnvironments } from "../../src/utils/get-environment-variables.js"; +import { getConnectorImage } from "../../src/utils/index.js"; +import { getCertificationImage } from "../../src/utils/index.js"; dotenv.config(); +const ATLAN_INSTANCE_URL = process.env.ATLAN_INSTANCE_URL; +const { IS_DEV } = process.env; export default class GitLabIntegration extends IntegrationInterface { constructor(token) { @@ -19,6 +23,7 @@ export default class GitLabIntegration extends IntegrationInterface { } async run() { + //Done console.log("Run Gitlab"); const timeStart = Date.now(); @@ -30,9 +35,10 @@ export default class GitLabIntegration extends IntegrationInterface { const { CI_PROJECT_PATH, CI_MERGE_REQUEST_IID } = process.env; if (!(await this.authIntegration({ gitlab }))) + //Done throw { message: "Wrong API Token" }; - const { state, web_url } = await gitlab.MergeRequests.show( + const { state, web_url, source_branch } = await gitlab.MergeRequests.show( CI_PROJECT_PATH, CI_MERGE_REQUEST_IID ); @@ -40,9 +46,16 @@ export default class GitLabIntegration extends IntegrationInterface { let total_assets = 0; if (state === "opened") { - total_assets = await this.printDownstreamAssets({ gitlab }); + total_assets = await this.printDownstreamAssets({ + gitlab, + source_branch, + }); } else if (state === "merged") { - total_assets = await this.setResourceOnAsset({ gitlab, web_url }); + total_assets = await this.setResourceOnAsset({ + gitlab, + web_url, + source_branch, + }); } if (total_assets !== 0) @@ -52,7 +65,8 @@ export default class GitLabIntegration extends IntegrationInterface { }); } - async printDownstreamAssets({ gitlab }) { + async printDownstreamAssets({ gitlab, source_branch }) { + //Done // Implementation for printing impact on GitHub // Use this.token to access the token const changedFiles = await this.getChangedFiles({ gitlab }); //Complete @@ -68,14 +82,24 @@ export default class GitLabIntegration extends IntegrationInterface { filePath, headSHA, }); + + const environments = getGitLabEnvironments(); + + let environment = null; + for (const [baseBranchName, environmentName] of environments) { + if (baseBranchName === source_branch) { + environment = environmentName; + break; + } + } + const asset = await getAsset({ - //Incomplete + //Complete name: assetName, sendSegmentEventOfIntegration: this.sendSegmentEventOfIntegration, + environment: environment, + integration: "gitlab", }); - // TODO :- When we call getAsset we are always sending segment event to github. We have to resolve this. - // Either we can pass that function to getAsset function to resolve this. Or check for better alternatives. - // If we pass the function we can simply write getAsset({name: assetName}, this.sendSegmentEvent) if (asset.error) { comments += asset.error; @@ -83,12 +107,19 @@ export default class GitLabIntegration extends IntegrationInterface { continue; } + //Cross-check this part once with Jaagrav. + + const totalModifiedFiles = changedFiles.filter( + (i) => i.status === "modified" + ).length; + const { guid } = asset.attributes.sqlAsset; const timeStart = Date.now(); const downstreamAssets = await getDownstreamAssets( - //Incomplete + //Done asset, guid, + totalModifiedFiles, this.sendSegmentEventOfIntegration, "gitlab" ); @@ -143,10 +174,11 @@ ${comments}`; return totalChangedFiles; } - async setResourceOnAsset({ gitlab, web_url }) { + async setResourceOnAsset({ gitlab, web_url, source_branch }) { + //Done // Implementation for setting resources on GitHub // Use this.token to access the token - const changedFiles = await this.getChangedFiles({ gitlab }); + const changedFiles = await this.getChangedFiles({ gitlab }); //Done var totalChangedFiles = 0; if (changedFiles.length === 0) return; @@ -158,10 +190,23 @@ ${comments}`; filePath, headSHA, }); + + const environments = getGitLabEnvironments(); + + let environment = null; + for (const [baseBranchName, environmentName] of environments) { + if (baseBranchName === source_branch) { + environment = environmentName; + break; + } + } + const asset = await getAsset({ - //Incomplete + //Done name: assetName, sendSegmentEventOfIntegration: this.sendSegmentEventOfIntegration, + environment: environment, + integration: "gitlab", }); if (!asset) continue; @@ -170,6 +215,7 @@ ${comments}`; const { guid: tableAssetGuid } = asset.attributes.sqlAsset; await createResource( + //Done //Complete modelGuid, "Pull Request on GitLab", @@ -177,6 +223,7 @@ ${comments}`; this.sendSegmentEventOfIntegration ); await createResource( + //Done tableAssetGuid, "Pull Request on GitLab", web_url, @@ -187,6 +234,7 @@ ${comments}`; } const comment = await this.createIssueComment({ + //Done //Complete gitlab, content: `🎊 Congrats on the merge! @@ -201,19 +249,16 @@ This pull request has been added as a resource to all the assets modified. ✅ } async authIntegration({ gitlab }) { - //Incomplete - // Implement your auth logic here - // IMPORT ATLAN INSTANCE URL + //Done const response = await auth(); - // Inside this if condition check github secrets is mentioned + if (response?.status === 401) { + //Complete await this.createIssueComment( gitlab, - `We couldn't connect to your Atlan Instance, please make sure to set the valid Atlan Bearer Token as \`ATLAN_API_TOKEN\` as this repository's action secret. - -Atlan Instance URL: ${ATLAN_INSTANCE_URL} + `We couldn't connect to your Atlan Instance, please make sure to set the valid Atlan Bearer Token as \`ATLAN_API_TOKEN\` in your .gitlab-ci.yml file. -Set your repository action secrets [here](https://github.com/${context.payload.repository.full_name}/settings/secrets/actions). For more information on how to setup the Atlan dbt Action, please read the [setup documentation here](https://github.com/atlanhq/dbt-action/blob/main/README.md).` +Atlan Instance URL: ${ATLAN_INSTANCE_URL}` ); return false; } @@ -221,14 +266,14 @@ Set your repository action secrets [here](https://github.com/${context.payload.r if (response === undefined) { await this.createIssueComment( gitlab, - `We couldn't connect to your Atlan Instance, please make sure to set the valid Atlan Instance URL as \`ATLAN_INSTANCE_URL\` as this repository's action secret. + `We couldn't connect to your Atlan Instance, please make sure to set the valid Atlan Instance URL as \`ATLAN_INSTANCE_URL\` in your .gitlab-ci.yml file. Atlan Instance URL: ${ATLAN_INSTANCE_URL} Make sure your Atlan Instance URL is set in the following format. \`https://tenant.atlan.com\` -Set your repository action secrets [here](https://github.com/${context.payload.repository.full_name}/settings/secrets/actions). For more information on how to setup the Atlan dbt Action, please read the [setup documentation here](https://github.com/atlanhq/dbt-action/blob/main/README.md).` +` ); return false; } @@ -237,6 +282,7 @@ Set your repository action secrets [here](https://github.com/${context.payload.r } async createIssueComment({ + //Done //Complete gitlab, content, @@ -267,6 +313,7 @@ ${content}`; } async sendSegmentEventOfIntegration({ action, properties }) { + //Done //Complete // Implement your sendSegmentEvent logic here // IMPORT ATLAN_INSTANCE_URL. @@ -289,6 +336,7 @@ ${content}`; } async getChangedFiles({ gitlab }) { + //Done //Complete const { CI_PROJECT_PATH, CI_MERGE_REQUEST_IID } = process.env; @@ -297,7 +345,7 @@ ${content}`; CI_MERGE_REQUEST_IID ); var changedFiles = changes - .map(({ new_path }) => { + .map(({ new_path, old_path }) => { try { const [modelName] = new_path .match(/.*models\/(.*)\.sql/)[1] @@ -305,12 +353,32 @@ ${content}`; .reverse()[0] .split("."); + //Cross-check this with Jaagrav. ### if (modelName) { - return { - fileName: modelName, - filePath: new_path, - headSHA: diff_refs.head_sha, - }; + if (old_path === null) { + return { + fileName: modelName, + filePath: new_path, + headSHA: diff_refs.head_sha, + status: "added", + }; + } else if (new_path !== old_path) { + // File is renamed or moved + return { + fileName: modelName, + filePath: new_path, + headSHA: diff_refs.head_sha, + status: "renamed_or_moved", + }; + } else { + // File is modified + return { + fileName: modelName, + filePath: new_path, + headSHA: diff_refs.head_sha, + status: "modified", + }; + } } } catch (e) {} }) @@ -329,6 +397,7 @@ ${content}`; } async getAssetName({ gitlab, fileName, filePath, headSHA }) { + //Done //Complete var regExp = /config\(.*alias=\'([^']+)\'.*\)/im; var fileContents = await this.getFileContents({ @@ -347,6 +416,7 @@ ${content}`; } async getFileContents({ gitlab, filePath, headSHA }) { + //Done //Complete const { CI_PROJECT_PATH } = process.env; const { content } = await gitlab.RepositoryFiles.show( @@ -360,6 +430,7 @@ ${content}`; } async checkCommentExists({ gitlab }) { + //Done //Complete const { CI_PROJECT_PATH, CI_MERGE_REQUEST_IID } = process.env; if (IS_DEV) return null; @@ -380,6 +451,7 @@ ${content}`; } async deleteComment({ gitlab, comment_id }) { + //Done //Complete const { CI_PROJECT_PATH, CI_MERGE_REQUEST_IID } = process.env; @@ -391,6 +463,7 @@ ${content}`; } async renderDownstreamAssetsComment({ asset, downstreamAssets }) { + //Done let impactedData = downstreamAssets.map( ({ displayText, guid, typeName, attributes, meanings }) => { let readableTypeName = typeName diff --git a/src/api/create-resource.js b/src/api/create-resource.js index 962394d..d3f51de 100644 --- a/src/api/create-resource.js +++ b/src/api/create-resource.js @@ -1,13 +1,12 @@ import { v4 as uuidv4 } from "uuid"; import fetch from "node-fetch"; -import { sendSegmentEvent } from "./index.js"; import stringify from "json-stringify-safe"; import { getAPIToken, getInstanceUrl } from "../utils/index.js"; const ATLAN_INSTANCE_URL = getInstanceUrl(); const ATLAN_API_TOKEN = getAPIToken(); -export default async function createResource( +export default async function createResource( //Done guid, name, link, diff --git a/src/api/get-asset.js b/src/api/get-asset.js index 7d8d219..a7e0adc 100644 --- a/src/api/get-asset.js +++ b/src/api/get-asset.js @@ -11,9 +11,11 @@ const ATLAN_API_TOKEN = core.getInput("ATLAN_API_TOKEN") || process.env.ATLAN_API_TOKEN; export default async function getAsset({ - //Incomplete + //Done name, sendSegmentEventOfIntegration, + environment, + integration, }) { var myHeaders = { Authorization: `Bearer ${ATLAN_API_TOKEN}`, @@ -23,7 +25,7 @@ export default async function getAsset({ var raw = stringify({ dsl: { from: 0, - size: 1, + size: 21, query: { bool: { must: [ @@ -42,6 +44,15 @@ export default async function getAsset({ "name.keyword": name, }, }, + ...(environment + ? [ + { + term: { + "assetDbtEnvironmentName.keyword": environment, + }, + }, + ] + : []), ], }, }, @@ -60,7 +71,15 @@ export default async function getAsset({ "ownerGroups", "classificationNames", "meanings", - "sqlAsset", + "dbtModelSqlAssets", + ], + relationAttributes: [ + "name", + "description", + "assetDbtProjectName", + "assetDbtEnvironmentName", + "connectorName", + "certificateStatus", ], }); @@ -83,8 +102,15 @@ export default async function getAsset({ }); }); - if (response?.entities?.length > 0) return response.entities[0]; - return { - error: `❌ Model with name ${name} not found

`, - }; + if (!response?.entities?.length) + return { + error: `❌ Model with name **${name}** could not be found or is deleted

`, + }; + + if (!response?.entities[0]?.attributes?.dbtModelSqlAssets?.length > 0) + return { + error: `❌ Model with name [${name}](${ATLAN_INSTANCE_URL}/assets/${response.entities[0].guid}/overview?utm_source=dbt_${integration}_action) does not materialise any asset

`, + }; + + return response.entities[0]; } diff --git a/src/api/get-classifications.js b/src/api/get-classifications.js index 282e2b9..c912590 100644 --- a/src/api/get-classifications.js +++ b/src/api/get-classifications.js @@ -1,34 +1,36 @@ import fetch from "node-fetch"; -import {sendSegmentEvent} from "./index.js"; -import stringify from 'json-stringify-safe'; -import {getAPIToken, getInstanceUrl} from "../utils/index.js"; +import { sendSegmentEvent } from "./index.js"; +import stringify from "json-stringify-safe"; +import { getAPIToken, getInstanceUrl } from "../utils/index.js"; -const ATLAN_INSTANCE_URL = - getInstanceUrl(); -const ATLAN_API_TOKEN = - getAPIToken(); +const ATLAN_INSTANCE_URL = getInstanceUrl(); +const ATLAN_API_TOKEN = getAPIToken(); -export default async function getClassifications() { - var myHeaders = { - Authorization: `Bearer ${ATLAN_API_TOKEN}`, - "Content-Type": "application/json", - }; +export default async function getClassifications({ + sendSegmentEventOfIntegration, +}) { + var myHeaders = { + Authorization: `Bearer ${ATLAN_API_TOKEN}`, + "Content-Type": "application/json", + }; - var requestOptions = { - method: 'GET', - headers: myHeaders, - redirect: 'follow' - }; + var requestOptions = { + method: "GET", + headers: myHeaders, + redirect: "follow", + }; - var response = await fetch( - `${ATLAN_INSTANCE_URL}/api/meta/types/typedefs?type=classification`, - requestOptions - ).then((e) => e.json()).catch(err => { - sendSegmentEvent("dbt_ci_action_failure", { - reason: 'failed_to_get_classifications', - msg: err - }); + var response = await fetch( + `${ATLAN_INSTANCE_URL}/api/meta/types/typedefs?type=classification`, + requestOptions + ) + .then((e) => e.json()) + .catch((err) => { + sendSegmentEventOfIntegration("dbt_ci_action_failure", { + reason: "failed_to_get_classifications", + msg: err, + }); }); - return response?.classificationDefs; -} \ No newline at end of file + return response?.classificationDefs; +} diff --git a/src/api/get-downstream-assets.js b/src/api/get-downstream-assets.js index fa2d611..5d7fc46 100644 --- a/src/api/get-downstream-assets.js +++ b/src/api/get-downstream-assets.js @@ -15,27 +15,38 @@ const ATLAN_INSTANCE_URL = const ATLAN_API_TOKEN = core.getInput("ATLAN_API_TOKEN") || process.env.ATLAN_API_TOKEN; -export default async function getDownstreamAssets( +export default async function getDownstreamAssets( //Done asset, guid, + totalModifiedFiles, sendSegmentEventOfIntegration, integration ) { - //Import SendSegmentEvent function var myHeaders = { authorization: `Bearer ${ATLAN_API_TOKEN}`, "content-type": "application/json", }; var raw = stringify({ - depth: 21, guid: guid, - hideProcess: true, - allowDeletedProcess: false, + size: Math.max(Math.ceil(ASSETS_LIMIT / totalModifiedFiles), 1), + from: 0, + depth: 21, + direction: "OUTPUT", entityFilters: { - attributeName: "__state", - operator: "eq", - attributeValue: "ACTIVE", + condition: "AND", + criterion: [ + { + attributeName: "__typeName", + operator: "not_contains", + attributeValue: "Process", + }, + { + attributeName: "__state", + operator: "eq", + attributeValue: "ACTIVE", + }, + ], }, attributes: [ "name", @@ -52,7 +63,8 @@ export default async function getDownstreamAssets( "classificationNames", "meanings", ], - direction: "OUTPUT", + excludeMeanings: false, + excludeClassifications: false, }); var requestOptions = { @@ -66,8 +78,7 @@ export default async function getDownstreamAssets( asset.attributes.connectorName )} [${asset.displayText}](${ATLAN_INSTANCE_URL}/assets/${ asset.guid - }?utm_source=dbt_${integration}_action) ${ - //Change it based on Integration name + }/overview?utm_source=dbt_${integration}_action) ${ asset.attributes?.certificateStatus ? getCertificationImage(asset.attributes.certificateStatus) : "" @@ -81,7 +92,7 @@ ${getImageURL( 15 )} [View lineage in Atlan](${ATLAN_INSTANCE_URL}/assets/${ asset.guid - }/lineage?utm_source=dbt_${integration}_action)`; + }/lineage/overview?utm_source=dbt_${integration}_action)`; sendSegmentEventOfIntegration("dbt_ci_action_failure", { reason: "failed_to_fetch_lineage", @@ -95,7 +106,7 @@ ${getImageURL( }; var response = await fetch( - `${ATLAN_INSTANCE_URL}/api/meta/lineage/getlineage`, + `${ATLAN_INSTANCE_URL}/api/meta/lineage/list`, requestOptions ) .then((e) => { @@ -113,11 +124,5 @@ ${getImageURL( if (response.error) return response; - if (!response?.relations) return []; - - const relations = response.relations.map(({ toEntityId }) => toEntityId); - - return relations - .filter((id, index) => relations.indexOf(id) === index) - .map((id) => response.guidEntityMap[id]); + return response; } diff --git a/src/api/segment.js b/src/api/segment.js index 9b6e11c..47cced1 100644 --- a/src/api/segment.js +++ b/src/api/segment.js @@ -23,7 +23,6 @@ export async function sendSegmentEvent(action, body) { }; var response = null; - if (!IS_DEV) { response = await fetch( `${ATLAN_INSTANCE_URL}/api/service/segment/track`, @@ -35,6 +34,8 @@ export async function sendSegmentEvent(action, body) { .catch((err) => { console.log("couldn't send segment event", err); }); + } else { + console.log("send segment event", action, body); } return response; diff --git a/src/utils/auth.js b/src/utils/auth.js index 388fa2b..17bae27 100644 --- a/src/utils/auth.js +++ b/src/utils/auth.js @@ -10,7 +10,7 @@ const ATLAN_API_TOKEN = core.getInput("ATLAN_API_TOKEN") || process.env.ATLAN_API_TOKEN; export async function auth() { - //Dont Change Anything + //Done var myHeaders = { authorization: `Bearer ${ATLAN_API_TOKEN}`, "content-type": "application/json", diff --git a/src/utils/get-environment-variables.js b/src/utils/get-environment-variables.js index ca7a1c3..ba30ead 100644 --- a/src/utils/get-environment-variables.js +++ b/src/utils/get-environment-variables.js @@ -3,19 +3,50 @@ import core from "@actions/core"; dotenv.config(); -const {IS_DEV, ATLAN_INSTANCE_URL, ATLAN_API_TOKEN, IGNORE_MODEL_ALIAS_MATCHING} = process.env; +const { + IS_DEV, + ATLAN_INSTANCE_URL, + ATLAN_API_TOKEN, + IGNORE_MODEL_ALIAS_MATCHING, +} = process.env; export const isDev = () => IS_DEV === "true"; export const getInstanceUrl = () => { - if (ATLAN_INSTANCE_URL) return new URL(ATLAN_INSTANCE_URL).origin; - return new URL(core.getInput("ATLAN_INSTANCE_URL")).origin; + if (ATLAN_INSTANCE_URL) return new URL(ATLAN_INSTANCE_URL).origin; + return new URL(core.getInput("ATLAN_INSTANCE_URL")).origin; }; export const getAPIToken = () => { - if (ATLAN_API_TOKEN) return ATLAN_API_TOKEN; - return core.getInput("ATLAN_API_TOKEN"); -} + if (ATLAN_API_TOKEN) return ATLAN_API_TOKEN; + return core.getInput("ATLAN_API_TOKEN"); +}; export const getEnvironments = () => { - return core.getInput('DBT_ENVIRONMENT_BRANCH_MAP') ? - core.getInput('DBT_ENVIRONMENT_BRANCH_MAP').trim()?.split('\n')?.map(i => i.split(':').map(i => i.trim())) : [] + return core.getInput("DBT_ENVIRONMENT_BRANCH_MAP") + ? core + .getInput("DBT_ENVIRONMENT_BRANCH_MAP") + .trim() + ?.split("\n") + ?.map((i) => i.split(":").map((i) => i.trim())) + : []; +}; +export const isIgnoreModelAliasMatching = () => + core.getInput("IGNORE_MODEL_ALIAS_MATCHING") === "true"; + +export function getGitLabEnvironments() { + const { DBT_ENVIRONMENT_BRANCH_MAP } = process.env; + + if (DBT_ENVIRONMENT_BRANCH_MAP) { + const environmentLines = DBT_ENVIRONMENT_BRANCH_MAP.split("\n"); + const environmentMap = {}; + + environmentLines.forEach((line) => { + const [environment, branch] = line.split(":").map((item) => item.trim()); + if (environment && branch) { + environmentMap[environment] = branch; + } + }); + + return environmentMap; + } else { + return {}; + } } -export const isIgnoreModelAliasMatching = () => core.getInput("IGNORE_MODEL_ALIAS_MATCHING") === "true"; \ No newline at end of file