Merge pull request #873 from atlanhq/DVX-641

Adds immediateNeighbors option and fixes missing lineage response details
atlanhq · Oct 3, 2024 · 5694e49 · 5694e49
2 parents 05cdffc + cec8a90
commit 5694e49
Show file tree

Hide file tree

Showing 520 changed files with 2,633 additions and 14 deletions.
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -9,13 +9,13 @@ log4j = "2.24.1"
 wiremock = "3.9.1"
 jnanoid = "2.0.0"
 numaflow = "0.8.0"
-awssdk = "2.28.13"
+awssdk = "2.28.14"
 gcs = "26.47.0"
 system-stubs = "2.1.7"
 fastcsv = "3.3.1"
 poi = "5.3.0"
 parsson = "1.1.7"
-simplejavamail = "8.12.0"
+simplejavamail = "8.12.1"
 swagger = "2.1.22"
 jsonpath = "2.9.0"
 commons-compress = "1.27.1"

diff --git a/integration-tests/src/test/java/com/atlan/java/sdk/LineageTest.java b/integration-tests/src/test/java/com/atlan/java/sdk/LineageTest.java
@@ -183,13 +183,17 @@ void fetchLineageListStart() throws AtlanException {
         assertEquals(response.getEntityCount(), 4);
         Asset one = response.getAssets().get(0);
         assertTrue(one instanceof LineageProcess);
+        assertEquals(one.getDepth(), 1);
         one = response.getAssets().get(1);
         assertTrue(one instanceof MaterializedView);
         assertEquals(one.getGuid(), mview.getGuid());
+        assertEquals(one.getDepth(), 1);
         one = response.getAssets().get(2);
         assertTrue(one instanceof LineageProcess);
+        assertEquals(one.getDepth(), 2);
         one = response.getAssets().get(3);
         assertTrue(one instanceof View);
+        assertEquals(one.getDepth(), 2);
         assertEquals(one.getGuid(), view.getGuid());
         response = FluentLineage.builder(Atlan.getDefaultClient(), table)
                 .direction(AtlanLineageDirection.UPSTREAM)
@@ -201,6 +205,81 @@ void fetchLineageListStart() throws AtlanException {
         assertFalse(response.getHasMore());
     }
 
+    @Test(
+            groups = {"lineage.read.lineage"},
+            dependsOnGroups = {"lineage.create.lineage.*"})
+    void fetchLineageListStartDetailed() throws AtlanException {
+        LineageListResponse response = FluentLineage.builder(Atlan.getDefaultClient(), table)
+                .toRequestBuilder()
+                .immediateNeighbors(true)
+                .build()
+                .fetch();
+        assertNotNull(response);
+        assertNotNull(response.getAssets());
+        assertEquals(response.getAssets().size(), 5);
+        assertEquals(response.getEntityCount(), 4);
+        Asset one = response.getAssets().get(0);
+        assertTrue(one instanceof Table);
+        assertEquals(one.getGuid(), table.getGuid());
+        assertEquals(one.getDepth(), 0);
+        assertNull(one.getImmediateUpstream());
+        assertNotNull(one.getImmediateDownstream());
+        assertEquals(one.getImmediateDownstream().size(), 1);
+        assertEquals(one.getImmediateDownstream().get(0).getGuid(), mview.getGuid());
+        one = response.getAssets().get(1);
+        assertTrue(one instanceof LineageProcess);
+        assertEquals(one.getGuid(), start.getGuid());
+        assertEquals(one.getDepth(), 1);
+        assertNotNull(one.getImmediateUpstream());
+        assertTrue(one.getImmediateUpstream().isEmpty());
+        assertNotNull(one.getImmediateDownstream());
+        assertEquals(one.getImmediateDownstream().size(), 1);
+        assertEquals(one.getImmediateDownstream().get(0).getGuid(), end.getGuid());
+        one = response.getAssets().get(2);
+        assertTrue(one instanceof MaterializedView);
+        assertEquals(one.getGuid(), mview.getGuid());
+        assertEquals(one.getDepth(), 1);
+        assertNotNull(one.getImmediateUpstream());
+        assertEquals(one.getImmediateUpstream().size(), 1);
+        assertEquals(one.getImmediateUpstream().get(0).getGuid(), table.getGuid());
+        assertNotNull(one.getImmediateDownstream());
+        assertEquals(one.getImmediateDownstream().size(), 1);
+        assertEquals(one.getImmediateDownstream().get(0).getGuid(), view.getGuid());
+        one = response.getAssets().get(3);
+        assertTrue(one instanceof LineageProcess);
+        assertEquals(one.getGuid(), end.getGuid());
+        assertEquals(one.getDepth(), 2);
+        assertNotNull(one.getImmediateUpstream());
+        assertEquals(one.getImmediateUpstream().size(), 1);
+        assertEquals(one.getImmediateUpstream().get(0).getGuid(), start.getGuid());
+        assertNotNull(one.getImmediateDownstream());
+        assertTrue(one.getImmediateDownstream().isEmpty());
+        one = response.getAssets().get(4);
+        assertTrue(one instanceof View);
+        assertEquals(one.getDepth(), 2);
+        assertEquals(one.getGuid(), view.getGuid());
+        assertNotNull(one.getImmediateUpstream());
+        assertEquals(one.getImmediateUpstream().size(), 1);
+        assertEquals(one.getImmediateUpstream().get(0).getGuid(), mview.getGuid());
+        assertNull(one.getImmediateDownstream());
+        response = FluentLineage.builder(Atlan.getDefaultClient(), table)
+                .direction(AtlanLineageDirection.UPSTREAM)
+                .toRequestBuilder()
+                .immediateNeighbors(true)
+                .build()
+                .fetch();
+        assertNotNull(response);
+        assertNotNull(response.getAssets());
+        assertEquals(response.getAssets().size(), 1);
+        one = response.getAssets().get(0);
+        assertTrue(one instanceof Table);
+        assertEquals(one.getGuid(), table.getGuid());
+        assertEquals(one.getDepth(), 0);
+        assertNull(one.getImmediateUpstream());
+        assertNull(one.getImmediateDownstream());
+        assertFalse(response.getHasMore());
+    }
+
     @Test(
             groups = {"lineage.read.lineage"},
             dependsOnGroups = {"lineage.create.lineage.*"})
@@ -221,14 +300,18 @@ void testLineageListIterators() throws AtlanException {
         assertEquals(results.size(), 4);
         Asset one = results.get(0);
         assertTrue(one instanceof LineageProcess);
+        assertEquals(one.getDepth(), 1);
         one = results.get(1);
         assertTrue(one instanceof MaterializedView);
         assertEquals(one.getGuid(), mview.getGuid());
+        assertEquals(one.getDepth(), 1);
         one = results.get(2);
         assertTrue(one instanceof LineageProcess);
         one = results.get(3);
+        assertEquals(one.getDepth(), 2);
         assertTrue(one instanceof View);
         assertEquals(one.getGuid(), view.getGuid());
+        assertEquals(one.getDepth(), 2);
     }
 
     @Test(
@@ -257,9 +340,11 @@ void fetchLineageListMiddle() throws AtlanException {
         assertEquals(response.getEntityCount(), 2);
         one = response.getAssets().get(0);
         assertTrue(one instanceof LineageProcess);
+        assertEquals(one.getDepth(), 1);
         one = response.getAssets().get(1);
         assertTrue(one instanceof Table);
         assertEquals(one.getGuid(), table.getGuid());
+        assertEquals(one.getDepth(), 1);
     }
 
     @Test(
@@ -286,13 +371,17 @@ void fetchLineageListEnd() throws AtlanException {
         Asset one = response.getAssets().get(0);
         assertTrue(one instanceof LineageProcess);
         one = response.getAssets().get(1);
+        assertEquals(one.getDepth(), 1);
         assertTrue(one instanceof MaterializedView);
         assertEquals(one.getGuid(), mview.getGuid());
+        assertEquals(one.getDepth(), 1);
         one = response.getAssets().get(2);
         assertTrue(one instanceof LineageProcess);
+        assertEquals(one.getDepth(), 2);
         one = response.getAssets().get(3);
         assertTrue(one instanceof Table);
         assertEquals(one.getGuid(), table.getGuid());
+        assertEquals(one.getDepth(), 2);
     }
 
     @Test(

diff --git a/...lone/sdk-extension/src/main/java/com/probable/guacamole/model/assets/GuacamoleColumn.java b/...lone/sdk-extension/src/main/java/com/probable/guacamole/model/assets/GuacamoleColumn.java
@@ -119,6 +119,11 @@ public class GuacamoleColumn extends Asset
     @Attribute
     Long columnDuplicateValuesCountLong;
 
+    /** List of top-level upstream nested columns. */
+    @Attribute
+    @Singular("putColumnHierarchy")
+    List<Map<String, String>> columnHierarchy;
+
     /** TBC */
     @Attribute
     Histogram columnHistogram;
@@ -342,6 +347,10 @@ public class GuacamoleColumn extends Asset
     @Attribute
     Integer nestedColumnCount;
 
+    /** Order (position) in which this column appears in the nested Column (nest level starts at 1). */
+    @Attribute
+    String nestedColumnOrder;
+
     /** Nested columns that exist within this column. */
     @Attribute
     @Singular

diff --git a/sdk/src/main/java/com/atlan/api/UsersEndpoint.java b/sdk/src/main/java/com/atlan/api/UsersEndpoint.java
@@ -305,7 +305,9 @@ public AtlanUser getByUsername(String user) throws AtlanException {
      */
     public AtlanUser getByUsername(String user, RequestOptions options) throws AtlanException {
         UserResponse response = list("{\"username\":\"" + user + "\"}", options);
-        if (response != null && response.getRecords() != null) {
+        if (response != null
+                && response.getRecords() != null
+                && !response.getRecords().isEmpty()) {
             return response.getRecords().get(0);
         } else {
             return null;

diff --git a/sdk/src/main/java/com/atlan/cache/GroupCache.java b/sdk/src/main/java/com/atlan/cache/GroupCache.java
@@ -151,9 +151,7 @@ public void lookupByAlias(String alias) throws AtlanException {
     }
 
     private void cacheResponse(GroupResponse response) {
-        if (response != null
-                && response.getRecords() != null
-                && !response.getRecords().isEmpty()) {
+        if (response != null && response.getRecords() != null) {
             List<AtlanGroup> groups = response.getRecords();
             for (AtlanGroup group : groups) {
                 String groupId = group.getId();

diff --git a/sdk/src/main/java/com/atlan/cache/UserCache.java b/sdk/src/main/java/com/atlan/cache/UserCache.java
@@ -175,7 +175,9 @@ public void lookupByName(String username) throws AtlanException {
             }
         } else {
             AtlanUser user = usersEndpoint.getByUsername(username);
-            cache(user.getId(), username, user);
+            if (user != null) {
+                cache(user.getId(), username, user);
+            }
         }
     }
 
@@ -184,7 +186,9 @@ public void lookupByName(String username) throws AtlanException {
     public void lookupById(String id) throws AtlanException {
         try {
             AtlanUser user = usersEndpoint.getByGuid(id);
-            cache(id, user.getUsername(), user);
+            if (user != null) {
+                cache(id, user.getUsername(), user);
+            }
         } catch (NotFoundException e) {
             // Otherwise, check if it is an API token
             ApiToken token = apiTokensEndpoint.getByGuid(id);

diff --git a/sdk/src/main/java/com/atlan/generators/GeneratorConfig.java b/sdk/src/main/java/com/atlan/generators/GeneratorConfig.java
@@ -81,7 +81,8 @@ public class GeneratorConfig {
             Map.entry("matillionVersions", "addMatillionVersion"),
             Map.entry("cogniteTimeseries", "addCogniteTimeseries"),
             Map.entry("exceptionsForBusinessPolicy", "exceptionForBusinessPolicy"),
-            Map.entry("supersetChartFormData", "putSupersetChartFormData"));
+            Map.entry("supersetChartFormData", "putSupersetChartFormData"),
+            Map.entry("columnHierarchy", "putColumnHierarchy"));
 
     private static final Map<String, String> DEFAULT_ATTRIBUTE_RENAMES = Map.ofEntries(
             Map.entry("connectorName", "connectorType"),

diff --git a/sdk/src/main/java/com/atlan/model/assets/Asset.java b/sdk/src/main/java/com/atlan/model/assets/Asset.java
@@ -551,6 +551,10 @@ public abstract class Asset extends Reference implements IAsset, IReferenceable
     @Attribute
     String lastSyncWorkflowName;
 
+    /** Custom order for sorting purpose, managed by client */
+    @Attribute
+    String lexicographicalSortOrder;
+
     /** Links that are attached to this asset. */
     @Attribute
     @Singular
@@ -575,6 +579,11 @@ public abstract class Asset extends Reference implements IAsset, IReferenceable
     @Attribute
     String name;
 
+    /** Array of policy ids non-compliant to this asset */
+    @Attribute
+    @Singular
+    SortedSet<String> nonCompliantAssetPolicyGUIDs;
+
     /** Data products for which this asset is an output port. */
     @Attribute
     @Singular
@@ -827,10 +836,24 @@ public Set<String> getNullFields() {
 
     /**
      * Depth of this asset within lineage.
-     * Note: this will only available in assets retrieved via lineage, and will vary even for
+     * Note: this will only be available in assets retrieved via lineage, and will vary even for
      * the same asset depending on the starting point of the lineage requested.
      */
-    final Integer depth;
+    final Long depth;
+
+    /**
+     * Reference details about the asset(s) that are immediately upstream of this asset within lineage.
+     * Note: this will only be available in assets retrieved via lineage when {@code immediateNeighbors} is true,
+     * and could vary even for the same asset depending on the starting point and depth of the lineage requested.
+     */
+    final List<LineageRef> immediateUpstream;
+
+    /**
+     * Reference details about the asset(s) that are immediately downstream of this asset within lineage.
+     * Note: this will only be available in assets retrieved via lineage when {@code immediateNeighbors} is true,
+     * and could vary even for the same asset depending on the starting point and depth of the lineage requested.
+     */
+    final List<LineageRef> immediateDownstream;
 
     /**
      * The names of the Atlan tags that exist on the asset. This is not always returned, even by

diff --git a/sdk/src/main/java/com/atlan/model/assets/BIProcess.java b/sdk/src/main/java/com/atlan/model/assets/BIProcess.java
@@ -43,6 +43,10 @@ public class BIProcess extends Asset implements IBIProcess, ILineageProcess, IAs
     @Builder.Default
     String typeName = TYPE_NAME;
 
+    /** Additional Context of the ETL pipeline/notebook which creates the process. */
+    @Attribute
+    String additionalEtlContext;
+
     /** Tasks that exist within this process. */
     @Attribute
     @Singular

diff --git a/sdk/src/main/java/com/atlan/model/assets/Column.java b/sdk/src/main/java/com/atlan/model/assets/Column.java
@@ -93,6 +93,11 @@ public class Column extends Asset implements IColumn, ISQL, ICatalog, IAsset, IR
     @Attribute
     Long columnDuplicateValuesCountLong;
 
+    /** List of top-level upstream nested columns. */
+    @Attribute
+    @Singular("putColumnHierarchy")
+    List<Map<String, String>> columnHierarchy;
+
     /** List of values in a histogram that represents the contents of this column. */
     @Attribute
     Histogram columnHistogram;
@@ -305,6 +310,10 @@ public class Column extends Asset implements IColumn, ISQL, ICatalog, IAsset, IR
     @Attribute
     Integer nestedColumnCount;
 
+    /** Order (position) in which this column appears in the nested Column (nest level starts at 1). */
+    @Attribute
+    String nestedColumnOrder;
+
     /** Nested columns that exist within this column. */
     @Attribute
     @Singular

diff --git a/sdk/src/main/java/com/atlan/model/assets/ColumnProcess.java b/sdk/src/main/java/com/atlan/model/assets/ColumnProcess.java
@@ -44,6 +44,10 @@ public class ColumnProcess extends Asset implements IColumnProcess, ILineageProc
     @Builder.Default
     String typeName = TYPE_NAME;
 
+    /** Additional Context of the ETL pipeline/notebook which creates the process. */
+    @Attribute
+    String additionalEtlContext;
+
     /** Tasks that exist within this process. */
     @Attribute
     @Singular

diff --git a/sdk/src/main/java/com/atlan/model/assets/DbtColumnProcess.java b/sdk/src/main/java/com/atlan/model/assets/DbtColumnProcess.java
@@ -45,6 +45,10 @@ public class DbtColumnProcess extends Asset
     @Builder.Default
     String typeName = TYPE_NAME;
 
+    /** Additional Context of the ETL pipeline/notebook which creates the process. */
+    @Attribute
+    String additionalEtlContext;
+
     /** Tasks that exist within this process. */
     @Attribute
     @Singular

diff --git a/sdk/src/main/java/com/atlan/model/assets/DbtProcess.java b/sdk/src/main/java/com/atlan/model/assets/DbtProcess.java
@@ -44,6 +44,10 @@ public class DbtProcess extends Asset implements IDbtProcess, IDbt, ILineageProc
     @Builder.Default
     String typeName = TYPE_NAME;
 
+    /** Additional Context of the ETL pipeline/notebook which creates the process. */
+    @Attribute
+    String additionalEtlContext;
+
     /** Tasks that exist within this process. */
     @Attribute
     @Singular

diff --git a/sdk/src/main/java/com/atlan/model/assets/IADLS.java b/sdk/src/main/java/com/atlan/model/assets/IADLS.java
@@ -390,6 +390,9 @@ public interface IADLS {
     /** Name of the crawler that last synchronized this asset. */
     String getLastSyncWorkflowName();
 
+    /** Custom order for sorting purpose, managed by client */
+    String getLexicographicalSortOrder();
+
     /** Links that are attached to this asset. */
     SortedSet<ILink> getLinks();
 
@@ -405,6 +408,9 @@ public interface IADLS {
     /** Name of this asset. Fallback for display purposes, if displayName is empty. */
     String getName();
 
+    /** Array of policy ids non-compliant to this asset */
+    SortedSet<String> getNonCompliantAssetPolicyGUIDs();
+
     /** Tasks from which this asset is output. */
     SortedSet<IAirflowTask> getOutputFromAirflowTasks();
 

diff --git a/sdk/src/main/java/com/atlan/model/assets/IADLSAccount.java b/sdk/src/main/java/com/atlan/model/assets/IADLSAccount.java
@@ -464,6 +464,9 @@ public interface IADLSAccount {
     /** Name of the crawler that last synchronized this asset. */
     String getLastSyncWorkflowName();
 
+    /** Custom order for sorting purpose, managed by client */
+    String getLexicographicalSortOrder();
+
     /** Links that are attached to this asset. */
     SortedSet<ILink> getLinks();
 
@@ -479,6 +482,9 @@ public interface IADLSAccount {
     /** Name of this asset. Fallback for display purposes, if displayName is empty. */
     String getName();
 
+    /** Array of policy ids non-compliant to this asset */
+    SortedSet<String> getNonCompliantAssetPolicyGUIDs();
+
     /** Tasks from which this asset is output. */
     SortedSet<IAirflowTask> getOutputFromAirflowTasks();