Skip to content

Commit

Permalink
Adds on-demand mode vs full refresh to caching
Browse files Browse the repository at this point in the history 
Signed-off-by: Christopher Grote <cmgrote@users.noreply.github.com>
  • Loading branch information
@cmgrote
cmgrote committed Oct 2, 2024
1 parent 1f8b9c8 commit 92cdbd5
Show file tree
Hide file tree
Showing 10 changed files with 408 additions and 41 deletions.
30 changes: 30 additions & 0 deletions sdk/src/main/java/com/atlan/api/UsersEndpoint.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,36 @@ public List<AtlanUser> getByUsernames(List<String> users, RequestOptions options
}
}

/**
* Retrieves the user with a unique ID (GUID) that exactly matches the provided string.
*
* @param guid unique identifier by which to retrieve the user
* @return the user whose GUID matches the provided string, or null if there is none
* @throws AtlanException on any error during API invocation
*/
public AtlanUser getByGuid(String guid) throws AtlanException {
return getByGuid(guid, null);
}

/**
* Retrieves the user with a unique ID (GUID) that exactly matches the provided string.
*
* @param guid unique identifier by which to retrieve the user
* @param options to override default client settings
* @return the user whose GUID matches the provided string, or null if there is none
* @throws AtlanException on any error during API invocation
*/
public AtlanUser getByGuid(String guid, RequestOptions options) throws AtlanException {
UserResponse response = list("{\"id\":\"" + guid + "\"}", options);
if (response != null
&& response.getRecords() != null
&& !response.getRecords().isEmpty()) {
return response.getRecords().get(0);
} else {
return null;
}
}

/**
* Create a new user.
*
Expand Down
132 changes: 125 additions & 7 deletions sdk/src/main/java/com/atlan/cache/AbstractMassCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,34 +8,68 @@
import com.atlan.exception.NotFoundException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;

import com.atlan.model.core.AtlanObject;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;

/**
* Base class for reusable components that are common to all caches, where
* a cache is populated en-masse through batch refreshing.
*/
@Slf4j
public abstract class AbstractMassCache {
public abstract class AbstractMassCache<T extends AtlanObject> {

protected final Map<String, String> mapIdToName = new ConcurrentHashMap<>();
protected final Map<String, String> mapNameToId = new ConcurrentHashMap<>();
protected final Map<String, T> mapIdToObject = new ConcurrentHashMap<>();

private final Map<String, String> mapIdToName = new ConcurrentHashMap<>();
private final Map<String, String> mapNameToId = new ConcurrentHashMap<>();
/** Whether to refresh the cache by retrieving all objects up-front (true) or lazily, on-demand (false). */
@Getter
protected AtomicBoolean bulkRefresh = new AtomicBoolean(true);

/**
* Logic to refresh the cache of objects from Atlan.
*
* @throws AtlanException on any error communicating with Atlan to refresh the cache of objects
*/
public abstract void refreshCache() throws AtlanException;
public synchronized void refreshCache() throws AtlanException {
mapIdToName.clear();
mapNameToId.clear();
mapIdToObject.clear();
}

/**
* Logic to look up a single object for the cache.
*
* @param id unique internal identifier for the object
* @throws AtlanException on any error communicating with Atlan
*/
public abstract void lookupById(String id) throws AtlanException;

/**
* Logic to look up a single object for the cache.
*
* @param name unique name for the object
* @throws AtlanException on any error communicating with Atlan
*/
public abstract void lookupByName(String name) throws AtlanException;

/**
* Add an entry to the cache
*
* @param id Atlan-internal ID
* @param name human-readable name
* @param object the object to cache (if any)
*/
protected void cache(String id, String name) {
protected void cache(String id, String name, T object) {
mapIdToName.put(id, name);
mapNameToId.put(name, id);
if (object != null) {
mapIdToObject.put(id, object);
}
}

/**
Expand Down Expand Up @@ -88,7 +122,11 @@ public String getIdForName(String name, boolean allowRefresh) throws AtlanExcept
String id = mapNameToId.get(name);
if (id == null && allowRefresh) {
// If not found, refresh the cache and look again (could be stale)
refreshCache();
if (bulkRefresh.get()) {
refreshCache();
} else {
lookupByName(name);
}
id = mapNameToId.get(name);
}
if (id == null) {
Expand Down Expand Up @@ -128,7 +166,11 @@ public String getNameForId(String id, boolean allowRefresh) throws AtlanExceptio
String name = mapIdToName.get(id);
if (name == null && allowRefresh) {
// If not found, refresh the cache and look again (could be stale)
refreshCache();
if (bulkRefresh.get()) {
refreshCache();
} else {
lookupById(id);
}
name = mapIdToName.get(id);
}
if (name == null) {
Expand All @@ -139,4 +181,80 @@ public String getNameForId(String id, boolean allowRefresh) throws AtlanExceptio
throw new InvalidRequestException(ErrorCode.MISSING_ID);
}
}

/**
* Retrieve the actual object by Atlan-internal ID string.
*
* @param id Atlan-internal ID string
* @return the object with that ID
* @throws AtlanException on any API communication problem if the cache needs to be refreshed
* @throws NotFoundException if the object cannot be found (does not exist) in Atlan
* @throws InvalidRequestException if no ID was provided for the object to retrieve
*/
public T getById(String id) throws AtlanException {
return getById(id, true);
}

/**
* Retrieve the actual object by Atlan-internal ID string.
*
* @param id Atlan-internal ID string
* @param allowRefresh whether to allow a refresh of the cache (true) or not (false)
* @return the object with that ID
* @throws AtlanException on any API communication problem if the cache needs to be refreshed
* @throws NotFoundException if the object cannot be found (does not exist) in Atlan
* @throws InvalidRequestException if no ID was provided for the object to retrieve
*/
public T getById(String id, boolean allowRefresh) throws AtlanException {
if (id != null && !id.isEmpty()) {
T result = mapIdToObject.get(id);
if (result == null && allowRefresh) {
// If not found, refresh the cache and look again (could be stale)
if (bulkRefresh.get()) {
refreshCache();
} else {
lookupById(id);
}
result = mapIdToObject.get(id);
}
if (result == null) {
throw new NotFoundException(ErrorCode.NAME_NOT_FOUND_BY_ID, id);
}
return result;
} else {
throw new InvalidRequestException(ErrorCode.MISSING_ID);
}
}

/**
* Retrieve the actual object by human-readable name.
*
* @param name human-readable name of the object
* @return the object with that name
* @throws AtlanException on any API communication problem if the cache needs to be refreshed
* @throws NotFoundException if the object cannot be found (does not exist) in Atlan
* @throws InvalidRequestException if no name was provided for the object to retrieve
*/
public T getByName(String name) throws AtlanException {
return getByName(name, true);
}

/**
* Retrieve the actual object by human-readable name.
*
* @param name human-readable name of the object
* @param allowRefresh whether to allow a refresh of the cache (true) or not (false)
* @return the object with that name
* @throws AtlanException on any API communication problem if the cache needs to be refreshed
* @throws NotFoundException if the object cannot be found (does not exist) in Atlan
* @throws InvalidRequestException if no name was provided for the object to retrieve
*/
public T getByName(String name, boolean allowRefresh) throws AtlanException {
if (name != null && !name.isEmpty()) {
String id = getIdForName(name, allowRefresh);
return getById(id, false);
} else {
throw new InvalidRequestException(ErrorCode.MISSING_NAME);
}
}
}
17 changes: 15 additions & 2 deletions sdk/src/main/java/com/atlan/cache/AtlanTagCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* Atlan tags.
*/
@Slf4j
public class AtlanTagCache extends AbstractMassCache {
public class AtlanTagCache extends AbstractMassCache<AtlanTagDef> {

private Map<String, String> mapIdToSourceTagsAttrId = new ConcurrentHashMap<>();
private Set<String> deletedIds = ConcurrentHashMap.newKeySet();
Expand All @@ -32,6 +32,7 @@ public AtlanTagCache(TypeDefsEndpoint typeDefsEndpoint) {
/** {@inheritDoc} */
@Override
public synchronized void refreshCache() throws AtlanException {
super.refreshCache();
log.debug("Refreshing cache of Atlan tags...");
TypeDefResponse response =
typeDefsEndpoint.list(List.of(AtlanTypeCategory.ATLAN_TAG, AtlanTypeCategory.STRUCT));
Expand All @@ -46,7 +47,7 @@ public synchronized void refreshCache() throws AtlanException {
deletedNames = ConcurrentHashMap.newKeySet();
for (AtlanTagDef clsDef : tags) {
String typeId = clsDef.getName();
cache(typeId, clsDef.getDisplayName());
cache(typeId, clsDef.getDisplayName(), clsDef);
List<AttributeDef> attrs = clsDef.getAttributeDefs();
String sourceTagsId = "";
if (attrs != null && !attrs.isEmpty()) {
Expand All @@ -60,6 +61,18 @@ public synchronized void refreshCache() throws AtlanException {
}
}

/** {@inheritDoc} */
@Override
public void lookupByName(String name) {
// Nothing to do here, can only be looked up by internal ID
}

/** {@inheritDoc} */
@Override
public void lookupById(String id) {
// Since we can only look up in one direction, we should only allow bulk refresh
}

/** {@inheritDoc} */
@Override
public String getIdForName(String name, boolean allowRefresh) throws AtlanException {
Expand Down
26 changes: 18 additions & 8 deletions sdk/src/main/java/com/atlan/cache/CustomMetadataCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,8 @@
* custom metadata (including attributes).
*/
@Slf4j
public class CustomMetadataCache extends AbstractMassCache {
public class CustomMetadataCache extends AbstractMassCache<CustomMetadataDef> {

private Map<String, CustomMetadataDef> cacheById = new ConcurrentHashMap<>();
private Map<String, AttributeDef> attrCacheById = new ConcurrentHashMap<>();

private Map<String, Map<String, String>> mapAttrIdToName = new ConcurrentHashMap<>();
Expand All @@ -40,6 +39,7 @@ public CustomMetadataCache(TypeDefsEndpoint typeDefsEndpoint) {
@Override
public synchronized void refreshCache() throws AtlanException {
log.debug("Refreshing cache of custom metadata...");
super.refreshCache();
TypeDefResponse response =
typeDefsEndpoint.list(List.of(AtlanTypeCategory.CUSTOM_METADATA, AtlanTypeCategory.STRUCT));
if (response == null
Expand All @@ -48,15 +48,13 @@ public synchronized void refreshCache() throws AtlanException {
throw new AuthenticationException(ErrorCode.EXPIRED_API_TOKEN);
}
List<CustomMetadataDef> customMetadata = response.getCustomMetadataDefs();
cacheById = new ConcurrentHashMap<>();
attrCacheById = new ConcurrentHashMap<>();
mapAttrIdToName = new ConcurrentHashMap<>();
mapAttrNameToId = new ConcurrentHashMap<>();
archivedAttrIds = new ConcurrentHashMap<>();
for (CustomMetadataDef bmDef : customMetadata) {
String typeId = bmDef.getName();
cacheById.put(typeId, bmDef);
cache(typeId, bmDef.getDisplayName());
cache(typeId, bmDef.getDisplayName(), bmDef);
mapAttrIdToName.put(typeId, new ConcurrentHashMap<>());
mapAttrNameToId.put(typeId, new ConcurrentHashMap<>());
for (AttributeDef attributeDef : bmDef.getAttributeDefs()) {
Expand All @@ -80,6 +78,18 @@ public synchronized void refreshCache() throws AtlanException {
}
}

/** {@inheritDoc} */
@Override
public void lookupByName(String name) {
// Nothing to do here, can only be looked up by internal ID
}

/** {@inheritDoc} */
@Override
public void lookupById(String id) {
// Since we can only look up in one direction, we should only allow bulk refresh
}

/**
* Retrieve all the (active) custom metadata attributes. The map will be keyed by custom metadata set
* name, and the value will be a listing of all the (active) attributes within that set (with all the details
Expand Down Expand Up @@ -118,11 +128,11 @@ public Map<String, List<AttributeDef>> getAllCustomAttributes(boolean includeDel
*/
public Map<String, List<AttributeDef>> getAllCustomAttributes(boolean includeDeleted, boolean forceRefresh)
throws AtlanException {
if (cacheById.isEmpty() || forceRefresh) {
if (mapIdToObject.isEmpty() || forceRefresh) {
refreshCache();
}
Map<String, List<AttributeDef>> map = new HashMap<>();
for (Map.Entry<String, CustomMetadataDef> entry : cacheById.entrySet()) {
for (Map.Entry<String, CustomMetadataDef> entry : mapIdToObject.entrySet()) {
String typeId = entry.getKey();
String typeName = getNameForId(typeId);
CustomMetadataDef typeDef = entry.getValue();
Expand Down Expand Up @@ -296,7 +306,7 @@ public CustomMetadataDef getCustomMetadataDef(String setName) throws AtlanExcept
*/
public CustomMetadataDef getCustomMetadataDef(String setName, boolean allowRefresh) throws AtlanException {
String setId = getIdForName(setName, allowRefresh);
return cacheById.get(setId);
return mapIdToObject.get(setId);
}

/**
Expand Down
Loading

0 comments on commit 92cdbd5

Please sign in to comment.