FAI-14157 | Azure pipelines improvements (builds stream) (#1797)

* azure pipelines improvements * fix test * fixed state lower
faros-ai · Nov 26, 2024 · 9fa469c · 9fa469c
1 parent 5242c30
commit 9fa469c
Show file tree

Hide file tree

Showing 5 changed files with 32 additions and 36 deletions.
diff --git a/destinations/airbyte-faros-destination/src/converters/azurepipeline/builds.ts b/destinations/airbyte-faros-destination/src/converters/azurepipeline/builds.ts
@@ -36,13 +36,11 @@ export class Builds extends AzurePipelineConverter {
     const status = this.convertBuildState(build.result);
     const res: DestinationRecord[] = [];
 
-    const number = Number(build.buildNumber.replace(/\./g, ''));
     res.push({
       model: 'cicd_Build',
       record: {
         uid,
         name: build.buildNumber,
-        number,
         createdAt,
         startedAt,
         endedAt,

diff --git a/destinations/airbyte-faros-destination/src/converters/azurepipeline/common.ts b/destinations/airbyte-faros-destination/src/converters/azurepipeline/common.ts
@@ -75,29 +75,27 @@ export abstract class AzurePipelineConverter extends Converter {
     return this.azurePipelineConfig(ctx)?.application_mapping ?? {};
   }
 
-  convertBuildState(state: string | undefined): {
+  convertBuildState(result: string | undefined): {
     category: string;
     detail: string;
   } {
-    if (!state) {
+    if (!result) {
       return {category: BuildStateCategory.Unknown, detail: 'undefined'};
     }
-    const detail = state.toLowerCase();
-
     // Read more on Azure pipeline build result:
     // https://docs.microsoft.com/en-us/rest/api/azure/devops/build/builds/list?view=azure-devops-rest-6.0#buildresult
-    switch (detail) {
+    switch (result) {
       case 'canceled':
-        return {category: BuildStateCategory.Canceled, detail};
+        return {category: BuildStateCategory.Canceled, detail: result};
       case 'failed':
-        return {category: BuildStateCategory.Failed, detail};
+        return {category: BuildStateCategory.Failed, detail: result};
       case 'succeeded':
       case 'partiallySucceeded':
-        return {category: BuildStateCategory.Success, detail};
+        return {category: BuildStateCategory.Success, detail: result};
       case 'running':
-        return {category: BuildStateCategory.Running, detail};
+        return {category: BuildStateCategory.Running, detail: result};
       default:
-        return {category: BuildStateCategory.Custom, detail};
+        return {category: BuildStateCategory.Custom, detail: result};
     }
   }
 
@@ -167,28 +165,26 @@ export abstract class AzurePipelineConverter extends Converter {
     return result;
   }
 
-  convertBuildStepState(state: string | undefined): {
+  convertBuildStepState(result: string | undefined): {
     category: string;
     detail: string;
   } {
-    if (!state) {
+    if (!result) {
       return {category: BuildStateCategory.Unknown, detail: 'undefined'};
     }
-    const detail = state.toLowerCase();
     //https://docs.microsoft.com/en-us/rest/api/azure/devops/build/timeline/get?view=azure-devops-rest-6.0#taskresult
-    switch (detail) {
+    switch (result) {
       case 'abandoned':
       case 'canceled':
-        return {category: BuildStateCategory.Canceled, detail};
+        return {category: BuildStateCategory.Canceled, detail: result};
       case 'failed':
       case 'skipped':
-        return {category: BuildStateCategory.Failed, detail};
+        return {category: BuildStateCategory.Failed, detail: result};
       case 'succeeded':
-        return {category: BuildStateCategory.Success, detail};
       case 'succeededWithIssues':
-        return {category: BuildStateCategory.Queued, detail};
+        return {category: BuildStateCategory.Success, detail: result};
       default:
-        return {category: BuildStateCategory.Custom, detail};
+        return {category: BuildStateCategory.Custom, detail: result};
     }
   }
 

diff --git a/sources/azurepipeline-source/src/azurepipeline.ts b/sources/azurepipeline-source/src/azurepipeline.ts
@@ -213,14 +213,16 @@ export class AzurePipeline {
 
   async *getBuilds(
     project: string,
-    lastQueueTime?: string,
+    lastFinishTime?: string,
     logger?: AirbyteLogger
   ): AsyncGenerator<Build> {
-    const startTime = lastQueueTime ? new Date(lastQueueTime) : this.startDate;
+    const startTime = lastFinishTime
+      ? new Date(lastFinishTime)
+      : this.startDate;
     //https://docs.microsoft.com/en-us/rest/api/azure/devops/build/builds/list?view=azure-devops-rest-6.0
     //https://docs.microsoft.com/en-us/rest/api/azure/devops/build/builds/list?view=azure-devops-rest-6.0#buildqueryorder
     const params = {
-      queryOrder: 'queueTimeAscending',
+      queryOrder: 'finishTimeAscending',
       minTime: startTime.toISOString(),
       $top: this.pageSize,
     };

diff --git a/sources/azurepipeline-source/src/streams/builds.ts b/sources/azurepipeline-source/src/streams/builds.ts
@@ -10,7 +10,7 @@ import {AzurePipeline, AzurePipelineConfig} from '../azurepipeline';
 import {Build} from '../models';
 
 interface BuildState {
-  lastQueueTime: string;
+  lastFinishTime: string;
 }
 
 type StreamSlice = {
@@ -32,7 +32,7 @@ export class Builds extends AirbyteStreamBase {
     return 'id';
   }
   get cursorField(): string | string[] {
-    return 'queueTime';
+    return 'finishTime';
   }
   async *streamSlices(): AsyncGenerator<StreamSlice> {
     const azurePipeline = await AzurePipeline.instance(
@@ -51,30 +51,30 @@ export class Builds extends AirbyteStreamBase {
     streamSlice?: StreamSlice,
     streamState?: BuildState
   ): AsyncGenerator<Build> {
-    const lastQueueTime =
+    const lastFinishTime =
       syncMode === SyncMode.INCREMENTAL
-        ? streamState?.lastQueueTime
+        ? streamState?.lastFinishTime
         : undefined;
     const azurePipeline = await AzurePipeline.instance(
       this.config,
       this.logger
     );
     yield* azurePipeline.getBuilds(
       streamSlice.project,
-      lastQueueTime,
+      lastFinishTime,
       this.logger
     );
   }
   getUpdatedState(
     currentStreamState: BuildState,
     latestRecord: Build
   ): BuildState {
-    const lastQueueTime: Date = new Date(latestRecord.queueTime);
+    const lastFinishTime: Date = new Date(latestRecord.finishTime);
     return {
-      lastQueueTime:
-        lastQueueTime >= new Date(currentStreamState?.lastQueueTime || 0)
-          ? latestRecord.queueTime
-          : currentStreamState.lastQueueTime,
+      lastFinishTime:
+        lastFinishTime >= new Date(currentStreamState?.lastFinishTime || 0)
+          ? latestRecord.finishTime
+          : currentStreamState.lastFinishTime,
     };
   }
 }
diff --git a/sources/azurepipeline-source/test/index.test.ts b/sources/azurepipeline-source/test/index.test.ts
@@ -76,7 +76,7 @@ describe('index', () => {
       .query({
         'api-version': '6.0',
         $top: 100,
-        queryOrder: 'queueTimeAscending',
+        queryOrder: 'finishTimeAscending',
         minTime: WATERMARK,
       })
       .reply(200, {value: buildsResource});
@@ -97,7 +97,7 @@ describe('index', () => {
       SyncMode.INCREMENTAL,
       undefined,
       {project: 'proj1'},
-      {lastQueueTime: WATERMARK}
+      {lastFinishTime: WATERMARK}
     );
 
     const builds = [];