diff --git a/adapters/gateway.js b/adapters/gateway.js
index a933b65..df652c5 100644
--- a/adapters/gateway.js
+++ b/adapters/gateway.js
@@ -5,5 +5,5 @@ export async function runAction(token, integrationModule) {
return;
}
const integration = new integrationModule(token);
- await integration.run();
+ await integration.run(); // Add error part
}
diff --git a/adapters/integrations/github-integration.js b/adapters/integrations/github-integration.js
index 45ecf94..adfd11f 100644
--- a/adapters/integrations/github-integration.js
+++ b/adapters/integrations/github-integration.js
@@ -2,17 +2,22 @@
import dotenv from "dotenv"; // Check do we actually need it or not
import IntegrationInterface from "./contract/contract.js";
import github from "@actions/github";
+import { isIgnoreModelAliasMatching } from "../../src/utils/get-environment-variables.js";
+import { getConnectorImage } from "../../src/utils/get-image-url.js";
+import { getEnvironments } from "../../src/utils/get-environment-variables.js";
import {
getAsset,
getDownstreamAssets,
sendSegmentEvent,
createResource,
+ getClassifications,
} from "../../src/api/index.js";
-import {
- renderDownstreamAssetsComment,
- getImageURL,
- auth,
-} from "../../src/utils/index.js";
+import { getImageURL, auth } from "../../src/utils/index.js";
+import { isDev } from "../../src/utils/get-environment-variables.js";
+import { truncate } from "../../src/utils/create-comment.js";
+import { getInstanceUrl } from "../../src/utils/get-environment-variables.js";
+const IS_DEV = isDev();
+const ATLAN_INSTANCE_URL = getInstanceUrl();
dotenv.config();
@@ -22,6 +27,8 @@ export default class GitHubIntegration extends IntegrationInterface {
}
async run() {
+ //Done
+ //Complete
console.log("Run Github");
const timeStart = Date.now();
const { context } = github;
@@ -30,6 +37,8 @@ export default class GitHubIntegration extends IntegrationInterface {
const { state, merged } = pull_request;
if (!(await this.authIntegration({ octokit, context }))) {
+ //DONE
+ //Complete
throw { message: "Wrong API Token" };
}
@@ -43,6 +52,7 @@ export default class GitHubIntegration extends IntegrationInterface {
if (total_assets !== 0) {
this.sendSegmentEventOfIntegration("dbt_ci_action_run", {
+ //Complete
asset_count: total_assets,
total_time: Date.now() - timeStart,
});
@@ -50,30 +60,47 @@ export default class GitHubIntegration extends IntegrationInterface {
}
async printDownstreamAssets({ octokit, context }) {
- // Implementation for printing impact on GitHub
- // Use this.token to access the token
-
- const changedFiles = await this.getChangedFiles({ octokit, context }); // Complete
-
+ //Done
+ const changedFiles = await this.getChangedFiles(octokit, context); //Complete
let comments = ``;
let totalChangedFiles = 0;
- for (const { fileName, filePath } of changedFiles) {
- const assetName = await this.getAssetName({
- // Complete
+ for (const { fileName, filePath, status } of changedFiles) {
+ const aliasName = await this.getAssetName({
+ //Complete
octokit,
context,
fileName,
filePath,
});
+ const assetName = isIgnoreModelAliasMatching() ? fileName : aliasName; //Complete
+
+ const environments = getEnvironments();
+
+ let environment = null;
+ for (const [baseBranchName, environmentName] of environments) {
+ if (baseBranchName === context.payload.pull_request.base.ref) {
+ environment = environmentName;
+ break;
+ }
+ }
+
const asset = await getAsset({
- //Complete
+ //Done
name: assetName,
sendSegmentEventOfIntegration: this.sendSegmentEventOfIntegration,
+ environment: environment,
+ integration: "github",
});
- // TODO :- When we call getAsset we are always sending segment event to github. We have to resolve this.
- // Either we can pass that function to getAsset function to resolve this. Or check for better alternatives.
- // If we pass the function we can simply write getAsset({name: assetName}, this.sendSegmentEvent)
+
+ if (totalChangedFiles !== 0) comments += "\n\n---\n\n";
+
+ if (status === "added") {
+ comments += `### ${getConnectorImage("dbt")} ${fileName} 🆕
+Its a new model and not present in Atlan yet, you'll see the downstream impact for it after its present in Atlan.`;
+ totalChangedFiles++;
+ continue;
+ }
if (asset.error) {
comments += asset.error;
@@ -81,18 +108,18 @@ export default class GitHubIntegration extends IntegrationInterface {
continue;
}
- const { guid } = asset.attributes.sqlAsset;
+ const materialisedAsset = asset.attributes.dbtModelSqlAssets[0];
const timeStart = Date.now();
+ const totalModifiedFiles = changedFiles.filter(
+ (i) => i.status === "modified"
+ ).length;
const downstreamAssets = await getDownstreamAssets(
//Complete
asset,
- guid,
- this.sendSegmentEventOfIntegration,
- "github"
+ materialisedAsset.guid,
+ totalModifiedFiles
);
- if (totalChangedFiles !== 0) comments += "\n\n---\n\n";
-
if (downstreamAssets.error) {
comments += downstreamAssets.error;
totalChangedFiles++;
@@ -100,13 +127,21 @@ export default class GitHubIntegration extends IntegrationInterface {
}
this.sendSegmentEventOfIntegration("dbt_ci_action_downstream_unfurl", {
+ //Complete
asset_guid: asset.guid,
asset_type: asset.typeName,
- downstream_count: downstreamAssets.length,
+ downstream_count: downstreamAssets.entities.length,
total_fetch_time: Date.now() - timeStart,
});
- // WHICH CODE TO PICKUP ONE IN THE BETA OR IN THE GITLAB BRANCH
+
+ const classifications = await getClassifications({
+ //Complete
+ sendSegmentEventOfIntegration: this.sendSegmentEventOfIntegration,
+ });
+
const comment = await this.renderDownstreamAssetsComment({
+ //Done
+ //Complete
octokit,
context,
asset,
@@ -127,19 +162,20 @@ Here is your downstream impact analysis for **${totalChangedFiles} ${
${comments}`;
- const existingComment = await this.checkCommentExists({ octokit, context });
+ const existingComment = await this.checkCommentExists({ octokit, context }); //Complete
- if (totalChangedFiles > 0) {
+ if (totalChangedFiles > 0)
await this.createIssueComment({
+ //Complete
octokit,
context,
content: comments,
comment_id: existingComment?.id,
});
- }
if (totalChangedFiles === 0 && existingComment)
await this.deleteComment({
+ //Complete
octokit,
context,
comment_id: existingComment.id,
@@ -149,11 +185,9 @@ ${comments}`;
}
async setResourceOnAsset({ octokit, context }) {
- //COMPLETE
- // Implementation for setting resources on GitHub
- // Use this.token to access the token
-
- const changedFiles = await this.getChangedFiles({ octokit, context });
+ //Done
+ //Complete
+ const changedFiles = await this.getChangedFiles({ octokit, context }); //Complete
const { pull_request } = context.payload;
var totalChangedFiles = 0;
@@ -167,33 +201,54 @@ ${comments}`;
fileName,
filePath,
});
+
+ const environments = getEnvironments();
+
+ let environment = null;
+ for (const [baseBranchName, environmentName] of environments) {
+ if (baseBranchName === context.payload.pull_request.base.ref) {
+ environment = environmentName;
+ break;
+ }
+ }
+
const asset = await getAsset({
+ //Done
name: assetName,
sendSegmentEventOfIntegration: this.sendSegmentEventOfIntegration,
+ environment: environment,
+ integration: "github",
});
- if (!asset) continue;
+ if (asset.error) continue;
const { guid: modelGuid } = asset;
- const { guid: tableAssetGuid } = asset.attributes.sqlAsset;
+ const { guid: tableAssetGuid } =
+ asset?.attributes?.dbtModelSqlAssets?.[0];
+
+ if (modelGuid)
+ await createResource(
+ //Complete
+ modelGuid,
+ "Pull Request on GitHub",
+ pull_request.html_url,
+ this.sendSegmentEventOfIntegration
+ );
- await createResource(
- modelGuid,
- "Pull Request on GitHub",
- pull_request.html_url,
- this.sendSegmentEventOfIntegration
- );
- await createResource(
- tableAssetGuid,
- "Pull Request on GitHub",
- pull_request.html_url,
- this.sendSegmentEventOfIntegration
- );
+ if (tableAssetGuid)
+ await createResource(
+ //Complete
+ tableAssetGuid,
+ "Pull Request on GitHub",
+ pull_request.html_url,
+ this.sendSegmentEventOfIntegration
+ );
totalChangedFiles++;
}
const comment = await this.createIssueComment({
+ //Complete
octokit,
context,
content: `🎊 Congrats on the merge!
@@ -208,18 +263,15 @@ This pull request has been added as a resource to all the assets modified. ✅
}
async authIntegration({ octokit, context }) {
+ //DONE
//COMPLETE
- // IMPORT ATLAN INSTANCE
- // Implement your auth logic here
const response = await auth();
- // Need to change the route we are passing in createIssueComment.
- const existingComment = await checkCommentExists(octokit, context);
+ const existingComment = await this.checkCommentExists({ octokit, context });
console.log("Existing Comment", existingComment);
if (response?.status === 401) {
await this.createIssueComment(
- // Check how u are calling this createIssueComment function
octokit,
context,
`We couldn't connect to your Atlan Instance, please make sure to set the valid Atlan Bearer Token as \`ATLAN_API_TOKEN\` as this repository's action secret.
@@ -253,8 +305,8 @@ Set your repository action secrets [here](https://github.com/${context.payload.r
}
async sendSegmentEventOfIntegration({ action, properties }) {
- //COMPLETE
- // Implement your sendSegmentEvent logic here
+ //Done
+ //FullyComplete
// IMPORT ATLAN_INSTANCE_URL.
const domain = new URL(ATLAN_INSTANCE_URL).hostname;
@@ -274,7 +326,8 @@ Set your repository action secrets [here](https://github.com/${context.payload.r
}
async getChangedFiles({ octokit, context }) {
- //Complete
+ //Done
+ //FullyComplete
const { repository, pull_request } = context.payload,
owner = repository.owner.login,
repo = repository.name,
@@ -322,10 +375,15 @@ Set your repository action secrets [here](https://github.com/${context.payload.r
}
async getAssetName({ octokit, context, fileName, filePath }) {
- // Complete
+ //Done
+ // FullyComplete
var regExp =
/{{\s*config\s*\(\s*(?:[^,]*,)*\s*alias\s*=\s*['"]([^'"]+)['"](?:\s*,[^,]*)*\s*\)\s*}}/im;
- var fileContents = await this.getFileContents(octokit, context, filePath);
+ var fileContents = await this.getFileContents({
+ octokit,
+ context,
+ filePath,
+ });
if (fileContents) {
var matches = regExp.exec(fileContents);
@@ -339,7 +397,8 @@ Set your repository action secrets [here](https://github.com/${context.payload.r
}
async getFileContents({ octokit, context, filePath }) {
- // Complete
+ //Done
+ // FullyComplete
const { repository, pull_request } = context.payload,
owner = repository.owner.login,
repo = repository.name,
@@ -367,7 +426,8 @@ Set your repository action secrets [here](https://github.com/${context.payload.r
}
async checkCommentExists({ octokit, context }) {
- //COMPLETE
+ //Done
+ //FullyComplete
if (IS_DEV) return null;
const { pull_request } = context.payload;
@@ -387,7 +447,8 @@ Set your repository action secrets [here](https://github.com/${context.payload.r
}
async createIssueComment({
- // COMPLETE
+ //Done
+ // FullyComplete
octokit,
context,
content,
@@ -405,7 +466,7 @@ ${content}`;
body: content,
};
- console.log(content);
+ console.log(content, content.length);
if (IS_DEV) return content;
@@ -415,7 +476,8 @@ ${content}`;
}
async deleteComment({ octokit, context, comment_id }) {
- //COMPLETE
+ //Done
+ //FullyComplete
const { pull_request } = context.payload;
return octokit.rest.issues.deleteComment({
@@ -426,6 +488,8 @@ ${content}`;
}
async renderDownstreamAssetsComment({
+ //Done
+ //FullyComplete
octokit,
context,
asset,
diff --git a/adapters/integrations/gitlab-integration.js b/adapters/integrations/gitlab-integration.js
index 06a3ba4..35639c8 100644
--- a/adapters/integrations/gitlab-integration.js
+++ b/adapters/integrations/gitlab-integration.js
@@ -10,8 +10,12 @@ import {
sendSegmentEvent,
} from "../../src/api/index.js";
import { getImageURL, auth } from "../../src/utils/index.js";
-
+import { getGitLabEnvironments } from "../../src/utils/get-environment-variables.js";
+import { getConnectorImage } from "../../src/utils/index.js";
+import { getCertificationImage } from "../../src/utils/index.js";
dotenv.config();
+const ATLAN_INSTANCE_URL = process.env.ATLAN_INSTANCE_URL;
+const { IS_DEV } = process.env;
export default class GitLabIntegration extends IntegrationInterface {
constructor(token) {
@@ -19,6 +23,7 @@ export default class GitLabIntegration extends IntegrationInterface {
}
async run() {
+ //Done
console.log("Run Gitlab");
const timeStart = Date.now();
@@ -30,9 +35,10 @@ export default class GitLabIntegration extends IntegrationInterface {
const { CI_PROJECT_PATH, CI_MERGE_REQUEST_IID } = process.env;
if (!(await this.authIntegration({ gitlab })))
+ //Done
throw { message: "Wrong API Token" };
- const { state, web_url } = await gitlab.MergeRequests.show(
+ const { state, web_url, source_branch } = await gitlab.MergeRequests.show(
CI_PROJECT_PATH,
CI_MERGE_REQUEST_IID
);
@@ -40,9 +46,16 @@ export default class GitLabIntegration extends IntegrationInterface {
let total_assets = 0;
if (state === "opened") {
- total_assets = await this.printDownstreamAssets({ gitlab });
+ total_assets = await this.printDownstreamAssets({
+ gitlab,
+ source_branch,
+ });
} else if (state === "merged") {
- total_assets = await this.setResourceOnAsset({ gitlab, web_url });
+ total_assets = await this.setResourceOnAsset({
+ gitlab,
+ web_url,
+ source_branch,
+ });
}
if (total_assets !== 0)
@@ -52,7 +65,8 @@ export default class GitLabIntegration extends IntegrationInterface {
});
}
- async printDownstreamAssets({ gitlab }) {
+ async printDownstreamAssets({ gitlab, source_branch }) {
+ //Done
// Implementation for printing impact on GitHub
// Use this.token to access the token
const changedFiles = await this.getChangedFiles({ gitlab }); //Complete
@@ -68,14 +82,24 @@ export default class GitLabIntegration extends IntegrationInterface {
filePath,
headSHA,
});
+
+ const environments = getGitLabEnvironments();
+
+ let environment = null;
+ for (const [baseBranchName, environmentName] of environments) {
+ if (baseBranchName === source_branch) {
+ environment = environmentName;
+ break;
+ }
+ }
+
const asset = await getAsset({
- //Incomplete
+ //Complete
name: assetName,
sendSegmentEventOfIntegration: this.sendSegmentEventOfIntegration,
+ environment: environment,
+ integration: "gitlab",
});
- // TODO :- When we call getAsset we are always sending segment event to github. We have to resolve this.
- // Either we can pass that function to getAsset function to resolve this. Or check for better alternatives.
- // If we pass the function we can simply write getAsset({name: assetName}, this.sendSegmentEvent)
if (asset.error) {
comments += asset.error;
@@ -83,12 +107,19 @@ export default class GitLabIntegration extends IntegrationInterface {
continue;
}
+ //Cross-check this part once with Jaagrav.
+
+ const totalModifiedFiles = changedFiles.filter(
+ (i) => i.status === "modified"
+ ).length;
+
const { guid } = asset.attributes.sqlAsset;
const timeStart = Date.now();
const downstreamAssets = await getDownstreamAssets(
- //Incomplete
+ //Done
asset,
guid,
+ totalModifiedFiles,
this.sendSegmentEventOfIntegration,
"gitlab"
);
@@ -143,10 +174,11 @@ ${comments}`;
return totalChangedFiles;
}
- async setResourceOnAsset({ gitlab, web_url }) {
+ async setResourceOnAsset({ gitlab, web_url, source_branch }) {
+ //Done
// Implementation for setting resources on GitHub
// Use this.token to access the token
- const changedFiles = await this.getChangedFiles({ gitlab });
+ const changedFiles = await this.getChangedFiles({ gitlab }); //Done
var totalChangedFiles = 0;
if (changedFiles.length === 0) return;
@@ -158,10 +190,23 @@ ${comments}`;
filePath,
headSHA,
});
+
+ const environments = getGitLabEnvironments();
+
+ let environment = null;
+ for (const [baseBranchName, environmentName] of environments) {
+ if (baseBranchName === source_branch) {
+ environment = environmentName;
+ break;
+ }
+ }
+
const asset = await getAsset({
- //Incomplete
+ //Done
name: assetName,
sendSegmentEventOfIntegration: this.sendSegmentEventOfIntegration,
+ environment: environment,
+ integration: "gitlab",
});
if (!asset) continue;
@@ -170,6 +215,7 @@ ${comments}`;
const { guid: tableAssetGuid } = asset.attributes.sqlAsset;
await createResource(
+ //Done
//Complete
modelGuid,
"Pull Request on GitLab",
@@ -177,6 +223,7 @@ ${comments}`;
this.sendSegmentEventOfIntegration
);
await createResource(
+ //Done
tableAssetGuid,
"Pull Request on GitLab",
web_url,
@@ -187,6 +234,7 @@ ${comments}`;
}
const comment = await this.createIssueComment({
+ //Done
//Complete
gitlab,
content: `🎊 Congrats on the merge!
@@ -201,19 +249,16 @@ This pull request has been added as a resource to all the assets modified. ✅
}
async authIntegration({ gitlab }) {
- //Incomplete
- // Implement your auth logic here
- // IMPORT ATLAN INSTANCE URL
+ //Done
const response = await auth();
- // Inside this if condition check github secrets is mentioned
+
if (response?.status === 401) {
+ //Complete
await this.createIssueComment(
gitlab,
- `We couldn't connect to your Atlan Instance, please make sure to set the valid Atlan Bearer Token as \`ATLAN_API_TOKEN\` as this repository's action secret.
-
-Atlan Instance URL: ${ATLAN_INSTANCE_URL}
+ `We couldn't connect to your Atlan Instance, please make sure to set the valid Atlan Bearer Token as \`ATLAN_API_TOKEN\` in your .gitlab-ci.yml file.
-Set your repository action secrets [here](https://github.com/${context.payload.repository.full_name}/settings/secrets/actions). For more information on how to setup the Atlan dbt Action, please read the [setup documentation here](https://github.com/atlanhq/dbt-action/blob/main/README.md).`
+Atlan Instance URL: ${ATLAN_INSTANCE_URL}`
);
return false;
}
@@ -221,14 +266,14 @@ Set your repository action secrets [here](https://github.com/${context.payload.r
if (response === undefined) {
await this.createIssueComment(
gitlab,
- `We couldn't connect to your Atlan Instance, please make sure to set the valid Atlan Instance URL as \`ATLAN_INSTANCE_URL\` as this repository's action secret.
+ `We couldn't connect to your Atlan Instance, please make sure to set the valid Atlan Instance URL as \`ATLAN_INSTANCE_URL\` in your .gitlab-ci.yml file.
Atlan Instance URL: ${ATLAN_INSTANCE_URL}
Make sure your Atlan Instance URL is set in the following format.
\`https://tenant.atlan.com\`
-Set your repository action secrets [here](https://github.com/${context.payload.repository.full_name}/settings/secrets/actions). For more information on how to setup the Atlan dbt Action, please read the [setup documentation here](https://github.com/atlanhq/dbt-action/blob/main/README.md).`
+`
);
return false;
}
@@ -237,6 +282,7 @@ Set your repository action secrets [here](https://github.com/${context.payload.r
}
async createIssueComment({
+ //Done
//Complete
gitlab,
content,
@@ -267,6 +313,7 @@ ${content}`;
}
async sendSegmentEventOfIntegration({ action, properties }) {
+ //Done
//Complete
// Implement your sendSegmentEvent logic here
// IMPORT ATLAN_INSTANCE_URL.
@@ -289,6 +336,7 @@ ${content}`;
}
async getChangedFiles({ gitlab }) {
+ //Done
//Complete
const { CI_PROJECT_PATH, CI_MERGE_REQUEST_IID } = process.env;
@@ -297,7 +345,7 @@ ${content}`;
CI_MERGE_REQUEST_IID
);
var changedFiles = changes
- .map(({ new_path }) => {
+ .map(({ new_path, old_path }) => {
try {
const [modelName] = new_path
.match(/.*models\/(.*)\.sql/)[1]
@@ -305,12 +353,32 @@ ${content}`;
.reverse()[0]
.split(".");
+ //Cross-check this with Jaagrav. ###
if (modelName) {
- return {
- fileName: modelName,
- filePath: new_path,
- headSHA: diff_refs.head_sha,
- };
+ if (old_path === null) {
+ return {
+ fileName: modelName,
+ filePath: new_path,
+ headSHA: diff_refs.head_sha,
+ status: "added",
+ };
+ } else if (new_path !== old_path) {
+ // File is renamed or moved
+ return {
+ fileName: modelName,
+ filePath: new_path,
+ headSHA: diff_refs.head_sha,
+ status: "renamed_or_moved",
+ };
+ } else {
+ // File is modified
+ return {
+ fileName: modelName,
+ filePath: new_path,
+ headSHA: diff_refs.head_sha,
+ status: "modified",
+ };
+ }
}
} catch (e) {}
})
@@ -329,6 +397,7 @@ ${content}`;
}
async getAssetName({ gitlab, fileName, filePath, headSHA }) {
+ //Done
//Complete
var regExp = /config\(.*alias=\'([^']+)\'.*\)/im;
var fileContents = await this.getFileContents({
@@ -347,6 +416,7 @@ ${content}`;
}
async getFileContents({ gitlab, filePath, headSHA }) {
+ //Done
//Complete
const { CI_PROJECT_PATH } = process.env;
const { content } = await gitlab.RepositoryFiles.show(
@@ -360,6 +430,7 @@ ${content}`;
}
async checkCommentExists({ gitlab }) {
+ //Done
//Complete
const { CI_PROJECT_PATH, CI_MERGE_REQUEST_IID } = process.env;
if (IS_DEV) return null;
@@ -380,6 +451,7 @@ ${content}`;
}
async deleteComment({ gitlab, comment_id }) {
+ //Done
//Complete
const { CI_PROJECT_PATH, CI_MERGE_REQUEST_IID } = process.env;
@@ -391,6 +463,7 @@ ${content}`;
}
async renderDownstreamAssetsComment({ asset, downstreamAssets }) {
+ //Done
let impactedData = downstreamAssets.map(
({ displayText, guid, typeName, attributes, meanings }) => {
let readableTypeName = typeName
diff --git a/src/api/create-resource.js b/src/api/create-resource.js
index 962394d..d3f51de 100644
--- a/src/api/create-resource.js
+++ b/src/api/create-resource.js
@@ -1,13 +1,12 @@
import { v4 as uuidv4 } from "uuid";
import fetch from "node-fetch";
-import { sendSegmentEvent } from "./index.js";
import stringify from "json-stringify-safe";
import { getAPIToken, getInstanceUrl } from "../utils/index.js";
const ATLAN_INSTANCE_URL = getInstanceUrl();
const ATLAN_API_TOKEN = getAPIToken();
-export default async function createResource(
+export default async function createResource( //Done
guid,
name,
link,
diff --git a/src/api/get-asset.js b/src/api/get-asset.js
index 7d8d219..a7e0adc 100644
--- a/src/api/get-asset.js
+++ b/src/api/get-asset.js
@@ -11,9 +11,11 @@ const ATLAN_API_TOKEN =
core.getInput("ATLAN_API_TOKEN") || process.env.ATLAN_API_TOKEN;
export default async function getAsset({
- //Incomplete
+ //Done
name,
sendSegmentEventOfIntegration,
+ environment,
+ integration,
}) {
var myHeaders = {
Authorization: `Bearer ${ATLAN_API_TOKEN}`,
@@ -23,7 +25,7 @@ export default async function getAsset({
var raw = stringify({
dsl: {
from: 0,
- size: 1,
+ size: 21,
query: {
bool: {
must: [
@@ -42,6 +44,15 @@ export default async function getAsset({
"name.keyword": name,
},
},
+ ...(environment
+ ? [
+ {
+ term: {
+ "assetDbtEnvironmentName.keyword": environment,
+ },
+ },
+ ]
+ : []),
],
},
},
@@ -60,7 +71,15 @@ export default async function getAsset({
"ownerGroups",
"classificationNames",
"meanings",
- "sqlAsset",
+ "dbtModelSqlAssets",
+ ],
+ relationAttributes: [
+ "name",
+ "description",
+ "assetDbtProjectName",
+ "assetDbtEnvironmentName",
+ "connectorName",
+ "certificateStatus",
],
});
@@ -83,8 +102,15 @@ export default async function getAsset({
});
});
- if (response?.entities?.length > 0) return response.entities[0];
- return {
- error: `❌ Model with name ${name} not found
`,
- };
+ if (!response?.entities?.length)
+ return {
+ error: `❌ Model with name **${name}** could not be found or is deleted
`,
+ };
+
+ if (!response?.entities[0]?.attributes?.dbtModelSqlAssets?.length > 0)
+ return {
+ error: `❌ Model with name [${name}](${ATLAN_INSTANCE_URL}/assets/${response.entities[0].guid}/overview?utm_source=dbt_${integration}_action) does not materialise any asset
`,
+ };
+
+ return response.entities[0];
}
diff --git a/src/api/get-classifications.js b/src/api/get-classifications.js
index 282e2b9..c912590 100644
--- a/src/api/get-classifications.js
+++ b/src/api/get-classifications.js
@@ -1,34 +1,36 @@
import fetch from "node-fetch";
-import {sendSegmentEvent} from "./index.js";
-import stringify from 'json-stringify-safe';
-import {getAPIToken, getInstanceUrl} from "../utils/index.js";
+import { sendSegmentEvent } from "./index.js";
+import stringify from "json-stringify-safe";
+import { getAPIToken, getInstanceUrl } from "../utils/index.js";
-const ATLAN_INSTANCE_URL =
- getInstanceUrl();
-const ATLAN_API_TOKEN =
- getAPIToken();
+const ATLAN_INSTANCE_URL = getInstanceUrl();
+const ATLAN_API_TOKEN = getAPIToken();
-export default async function getClassifications() {
- var myHeaders = {
- Authorization: `Bearer ${ATLAN_API_TOKEN}`,
- "Content-Type": "application/json",
- };
+export default async function getClassifications({
+ sendSegmentEventOfIntegration,
+}) {
+ var myHeaders = {
+ Authorization: `Bearer ${ATLAN_API_TOKEN}`,
+ "Content-Type": "application/json",
+ };
- var requestOptions = {
- method: 'GET',
- headers: myHeaders,
- redirect: 'follow'
- };
+ var requestOptions = {
+ method: "GET",
+ headers: myHeaders,
+ redirect: "follow",
+ };
- var response = await fetch(
- `${ATLAN_INSTANCE_URL}/api/meta/types/typedefs?type=classification`,
- requestOptions
- ).then((e) => e.json()).catch(err => {
- sendSegmentEvent("dbt_ci_action_failure", {
- reason: 'failed_to_get_classifications',
- msg: err
- });
+ var response = await fetch(
+ `${ATLAN_INSTANCE_URL}/api/meta/types/typedefs?type=classification`,
+ requestOptions
+ )
+ .then((e) => e.json())
+ .catch((err) => {
+ sendSegmentEventOfIntegration("dbt_ci_action_failure", {
+ reason: "failed_to_get_classifications",
+ msg: err,
+ });
});
- return response?.classificationDefs;
-}
\ No newline at end of file
+ return response?.classificationDefs;
+}
diff --git a/src/api/get-downstream-assets.js b/src/api/get-downstream-assets.js
index fa2d611..5d7fc46 100644
--- a/src/api/get-downstream-assets.js
+++ b/src/api/get-downstream-assets.js
@@ -15,27 +15,38 @@ const ATLAN_INSTANCE_URL =
const ATLAN_API_TOKEN =
core.getInput("ATLAN_API_TOKEN") || process.env.ATLAN_API_TOKEN;
-export default async function getDownstreamAssets(
+export default async function getDownstreamAssets( //Done
asset,
guid,
+ totalModifiedFiles,
sendSegmentEventOfIntegration,
integration
) {
- //Import SendSegmentEvent function
var myHeaders = {
authorization: `Bearer ${ATLAN_API_TOKEN}`,
"content-type": "application/json",
};
var raw = stringify({
- depth: 21,
guid: guid,
- hideProcess: true,
- allowDeletedProcess: false,
+ size: Math.max(Math.ceil(ASSETS_LIMIT / totalModifiedFiles), 1),
+ from: 0,
+ depth: 21,
+ direction: "OUTPUT",
entityFilters: {
- attributeName: "__state",
- operator: "eq",
- attributeValue: "ACTIVE",
+ condition: "AND",
+ criterion: [
+ {
+ attributeName: "__typeName",
+ operator: "not_contains",
+ attributeValue: "Process",
+ },
+ {
+ attributeName: "__state",
+ operator: "eq",
+ attributeValue: "ACTIVE",
+ },
+ ],
},
attributes: [
"name",
@@ -52,7 +63,8 @@ export default async function getDownstreamAssets(
"classificationNames",
"meanings",
],
- direction: "OUTPUT",
+ excludeMeanings: false,
+ excludeClassifications: false,
});
var requestOptions = {
@@ -66,8 +78,7 @@ export default async function getDownstreamAssets(
asset.attributes.connectorName
)} [${asset.displayText}](${ATLAN_INSTANCE_URL}/assets/${
asset.guid
- }?utm_source=dbt_${integration}_action) ${
- //Change it based on Integration name
+ }/overview?utm_source=dbt_${integration}_action) ${
asset.attributes?.certificateStatus
? getCertificationImage(asset.attributes.certificateStatus)
: ""
@@ -81,7 +92,7 @@ ${getImageURL(
15
)} [View lineage in Atlan](${ATLAN_INSTANCE_URL}/assets/${
asset.guid
- }/lineage?utm_source=dbt_${integration}_action)`;
+ }/lineage/overview?utm_source=dbt_${integration}_action)`;
sendSegmentEventOfIntegration("dbt_ci_action_failure", {
reason: "failed_to_fetch_lineage",
@@ -95,7 +106,7 @@ ${getImageURL(
};
var response = await fetch(
- `${ATLAN_INSTANCE_URL}/api/meta/lineage/getlineage`,
+ `${ATLAN_INSTANCE_URL}/api/meta/lineage/list`,
requestOptions
)
.then((e) => {
@@ -113,11 +124,5 @@ ${getImageURL(
if (response.error) return response;
- if (!response?.relations) return [];
-
- const relations = response.relations.map(({ toEntityId }) => toEntityId);
-
- return relations
- .filter((id, index) => relations.indexOf(id) === index)
- .map((id) => response.guidEntityMap[id]);
+ return response;
}
diff --git a/src/api/segment.js b/src/api/segment.js
index 9b6e11c..47cced1 100644
--- a/src/api/segment.js
+++ b/src/api/segment.js
@@ -23,7 +23,6 @@ export async function sendSegmentEvent(action, body) {
};
var response = null;
-
if (!IS_DEV) {
response = await fetch(
`${ATLAN_INSTANCE_URL}/api/service/segment/track`,
@@ -35,6 +34,8 @@ export async function sendSegmentEvent(action, body) {
.catch((err) => {
console.log("couldn't send segment event", err);
});
+ } else {
+ console.log("send segment event", action, body);
}
return response;
diff --git a/src/utils/auth.js b/src/utils/auth.js
index 388fa2b..17bae27 100644
--- a/src/utils/auth.js
+++ b/src/utils/auth.js
@@ -10,7 +10,7 @@ const ATLAN_API_TOKEN =
core.getInput("ATLAN_API_TOKEN") || process.env.ATLAN_API_TOKEN;
export async function auth() {
- //Dont Change Anything
+ //Done
var myHeaders = {
authorization: `Bearer ${ATLAN_API_TOKEN}`,
"content-type": "application/json",
diff --git a/src/utils/get-environment-variables.js b/src/utils/get-environment-variables.js
index ca7a1c3..ba30ead 100644
--- a/src/utils/get-environment-variables.js
+++ b/src/utils/get-environment-variables.js
@@ -3,19 +3,50 @@ import core from "@actions/core";
dotenv.config();
-const {IS_DEV, ATLAN_INSTANCE_URL, ATLAN_API_TOKEN, IGNORE_MODEL_ALIAS_MATCHING} = process.env;
+const {
+ IS_DEV,
+ ATLAN_INSTANCE_URL,
+ ATLAN_API_TOKEN,
+ IGNORE_MODEL_ALIAS_MATCHING,
+} = process.env;
export const isDev = () => IS_DEV === "true";
export const getInstanceUrl = () => {
- if (ATLAN_INSTANCE_URL) return new URL(ATLAN_INSTANCE_URL).origin;
- return new URL(core.getInput("ATLAN_INSTANCE_URL")).origin;
+ if (ATLAN_INSTANCE_URL) return new URL(ATLAN_INSTANCE_URL).origin;
+ return new URL(core.getInput("ATLAN_INSTANCE_URL")).origin;
};
export const getAPIToken = () => {
- if (ATLAN_API_TOKEN) return ATLAN_API_TOKEN;
- return core.getInput("ATLAN_API_TOKEN");
-}
+ if (ATLAN_API_TOKEN) return ATLAN_API_TOKEN;
+ return core.getInput("ATLAN_API_TOKEN");
+};
export const getEnvironments = () => {
- return core.getInput('DBT_ENVIRONMENT_BRANCH_MAP') ?
- core.getInput('DBT_ENVIRONMENT_BRANCH_MAP').trim()?.split('\n')?.map(i => i.split(':').map(i => i.trim())) : []
+ return core.getInput("DBT_ENVIRONMENT_BRANCH_MAP")
+ ? core
+ .getInput("DBT_ENVIRONMENT_BRANCH_MAP")
+ .trim()
+ ?.split("\n")
+ ?.map((i) => i.split(":").map((i) => i.trim()))
+ : [];
+};
+export const isIgnoreModelAliasMatching = () =>
+ core.getInput("IGNORE_MODEL_ALIAS_MATCHING") === "true";
+
+export function getGitLabEnvironments() {
+ const { DBT_ENVIRONMENT_BRANCH_MAP } = process.env;
+
+ if (DBT_ENVIRONMENT_BRANCH_MAP) {
+ const environmentLines = DBT_ENVIRONMENT_BRANCH_MAP.split("\n");
+ const environmentMap = {};
+
+ environmentLines.forEach((line) => {
+ const [environment, branch] = line.split(":").map((item) => item.trim());
+ if (environment && branch) {
+ environmentMap[environment] = branch;
+ }
+ });
+
+ return environmentMap;
+ } else {
+ return {};
+ }
}
-export const isIgnoreModelAliasMatching = () => core.getInput("IGNORE_MODEL_ALIAS_MATCHING") === "true";
\ No newline at end of file