FAI-12709: Refactor Vanta Source to pull generic vulns + remediations (…

…#1759)

* Replace GraphQL Queries with generic vulnerability endpoints

* URL mod and auth mod

* getting session token

* secrets

* Vanta converter refactor wip

* Add resources pull to vanta source

* API endpoints for resources

* Generate in a function

* typing

* Refactor destination to process new vulnerability and vulnerability remediation records.

* Add assets endpoint to retrieve repo name and image tags, implement query for cicd vulns

* Remove unused variable

* Remove updates and use vuln record type enum

* Split into two streams and add retryal with exponential backoff for 429 errors

* Remove extra data object.

* Use getAxiosResponse method

* Support incremental sync

* Bump max request retries to 8

* -Remove unused query to Faros graph
-Refactor Vanta api to use Faros axios client and support retry logic.
- Added schemas

* Set vanta property in instance method and remove unused import

* Add provisional logs and simplify remediations converter

* Fix vulnerability asset linking

* use assetType

Signed-off-by: Chalenge Masekera <5385518+chalenge@users.noreply.github.com>

* Query all repos and cicd artifacts in single queries and write identifiers + repo/cicd vulnerabilities on processing complete.

* Fix tests and remove unused stuff

* Fix sonnarcloud issues and remove vuln remediation records snapshot as it contains update "at" date and fails

* Refactor destination logic to reduce duplication and push assets both for vulns and remediations, to use in destination.

* Move things to common sec and fix severity and vulnerability identifier logic.

* add package lock

* Revert "add package lock"

This reverts commit 10c1403.

* fix build

* update package lock

* update lock

* Fix null asset error

* Review comments fixes

* Move functions to correct common file

* Fix dep and build

* Log vulnerabilities grouped.

* Add log and default retry Number fallback

* Remove log, add 60 seconds base default with exponential backoff for 429 and 500 statuses.

---------

Signed-off-by: Chalenge Masekera <5385518+chalenge@users.noreply.github.com>
Co-authored-by: omreego <omreeg@faros.ai>
Co-authored-by: Chalenge Masekera <5385518+chalenge@users.noreply.github.com>

destinations/airbyte-faros-destination/resources/cicdArtifactQueryByCommitSha.gql

@@ -1,5 +1,5 @@
-query cicdArtifactQueryByCommitSha($commitShas: [String], $limit: Int) {
-  cicd_Artifact(where: {uid: {_in: $commitShas}}, limit: $limit, distinct_on: uid) {
+query cicdArtifactQueryByCommitSha($commitShas: [String]) {
+  cicd_Artifact(where: {uid: {_in: $commitShas}}, distinct_on: uid) {
     uid
     repository {
       organization {

destinations/airbyte-faros-destination/resources/source-specific-configs/vanta.json

@@ -11,12 +11,12 @@
           "const": "Vanta",
           "order": 0
         },
-        "updateExistingVulnerabilities": {
-          "type": "boolean",
-          "title": "Update Faros Vulnerabilities to Resolved",
-          "default": true,
-          "order": 1
-        } 
+        "max_description_length": {
+          "type": "integer",
+          "title": "Max Description Length",
+          "description": "Defines the maximum length for descriptions before truncation",
+          "default": 1000
+        }
       }
     }
   ]

destinations/airbyte-faros-destination/resources/vcsRepositoryQuery.gql

@@ -1,5 +1,5 @@
-query vcsRepositoryQuery($vcsRepoNames: [String], $limit: Int) {
-  vcs_Repository(where: {name: {_in: $vcsRepoNames}}, limit: $limit, distinct_on: name) {
+query vcsRepositoryQuery($vcsRepoNames: [String]) {
+  vcs_Repository(where: {name: {_in: $vcsRepoNames}}, distinct_on: name) {
     organization {
       uid
       source

destinations/airbyte-faros-destination/src/converters/common/cicd.ts

@@ -0,0 +1,35 @@
+import {StreamContext} from '../converter';
+import {getQueryFromName} from '../vanta/utils';
+
+export interface CicdOrgKey {
+  uid: string;
+  source: string;
+}
+
+export interface CicdRepoKey {
+  organization: CicdOrgKey;
+  uid: string;
+}
+
+export interface ArtifactKey {
+  uid: string;
+  repository: CicdRepoKey;
+}
+
+const cicdArtifactQueryByCommitSha = getQueryFromName(
+  'cicdArtifactQueryByCommitSha'
+);
+
+export async function getCICDArtifactsFromCommitShas(
+  commitShas: string[],
+  ctx: StreamContext
+): Promise<ArtifactKey[] | null> {
+  const result = await ctx.farosClient.gql(
+    ctx.graph,
+    cicdArtifactQueryByCommitSha,
+    {
+      commitShas,
+    }
+  );
+  return result?.cicd_Artifact;
+}

destinations/airbyte-faros-destination/src/converters/common/sec.ts

@@ -1,5 +1,10 @@
 import {CategoryDetail} from './common';
 
+export interface VulnerabilityIdentifier {
+  uid: string;
+  type: CategoryDetail;
+}
+
 export class Vulnerability {
   // Mapping Qualitative Severity Ratings to CVSS v4.0 Severity Scores
   // using the upper bound of each rating

destinations/airbyte-faros-destination/src/converters/common/vcs.ts

@@ -1,7 +1,8 @@
 import {FileDiff} from 'faros-airbyte-common/common';
 import {uniq} from 'lodash';
 
-import {DestinationRecord} from '../converter';
+import {DestinationRecord, StreamContext} from '../converter';
+import {getQueryFromName} from '../vanta/utils';
 
 const NULL = '/dev/null';
 
@@ -137,3 +138,15 @@ export class FileCollector {
     }));
   }
 }
+
+const vcsRepositoryQuery = getQueryFromName('vcsRepositoryQuery');
+
+export async function getVCSRepositoriesFromNames(
+  vcsRepoNames: string[],
+  ctx: StreamContext
+): Promise<RepoKey[] | null> {
+  const result = await ctx.farosClient.gql(ctx.graph, vcsRepositoryQuery, {
+    vcsRepoNames,
+  });
+  return result?.vcs_Repository;
+}

destinations/airbyte-faros-destination/src/converters/vanta/common.ts

@@ -0,0 +1,45 @@
+import {AirbyteRecord} from 'faros-airbyte-cdk';
+import {VulnerableAssetSummary} from 'faros-airbyte-common/lib/vanta';
+
+import {Converter, StreamContext} from '../converter';
+import {looksLikeGitCommitSha} from './utils';
+
+export abstract class VantaConverter extends Converter {
+  source = 'Vanta';
+
+  /** All Vanta records should have id property */
+  id(record: AirbyteRecord): any {
+    return record?.record?.data?.id;
+  }
+
+  protected isVCSRepoVulnerability(
+    vulnerableAsset: VulnerableAssetSummary
+  ): boolean {
+    return vulnerableAsset.type === 'CODE_REPOSITORY';
+  }
+
+  protected isCICDArtifactVulnerability(
+    vulnerableAsset: VulnerableAssetSummary
+  ): boolean {
+    return this.getCommitSha(vulnerableAsset.imageTags)?.length > 0;
+  }
+
+  protected getCommitSha(imageTags?: string[]): string | null {
+    for (const imageTag of imageTags ?? []) {
+      if (looksLikeGitCommitSha(imageTag)) {
+        return imageTag;
+      }
+    }
+    return null;
+  }
+
+  protected logVulnerabilityWarnings(
+    ctx: StreamContext,
+    vulnerabilities: Set<string>,
+    message: string
+  ): void {
+    if (vulnerabilities.size > 0) {
+      ctx.logger.warn(`${message}: ${Array.from(vulnerabilities).join(', ')}`);
+    }
+  }
+}