diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index c110c6a6ac..3f0d5ee645 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -1187,9 +1187,12 @@ public static String[] tokenize(String url) { } Map convertToMap(List l) { + Set mergedUrls = MergedUrlsDao.instance.getMergedUrls(); Map ret = new HashMap<>(); for(SingleTypeInfo e: l) { - ret.put(e.composeKey(), e); + if(!mergedUrls.contains(new MergedUrls(e.getUrl(), e.getMethod(), e.getApiCollectionId()))) { + ret.put(e.composeKey(), e); + } } return ret; @@ -1910,7 +1913,7 @@ public void syncWithDB(boolean syncImmediately, boolean fetchAllSTI, SyncLimit s loggerMaker.infoAndAddToDb(res.getInserts().size() + " " +res.getUpserts().size(), LogDb.RUNTIME); } - + loggerMaker.infoAndAddToDb("adding " + writesForSampleData.size() + " updates for samples", LogDb.RUNTIME); if(writesForSampleData.size() > 0) { diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicyNew.java b/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicyNew.java index eb2e73f9ac..942da594ca 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicyNew.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicyNew.java @@ -2,8 +2,10 @@ import com.akto.dao.*; import com.akto.dao.context.Context; +import com.akto.dao.filter.MergedUrlsDao; import com.akto.dto.*; import com.akto.dto.ApiInfo.ApiInfoKey; +import com.akto.dto.filter.MergedUrls; import com.akto.dto.runtime_filters.RuntimeFilter; import com.akto.dto.testing.TestingEndpoints; import com.akto.dto.type.APICatalog; @@ -268,7 +270,15 @@ public static UpdateReturn getUpdates(Map apiInfoCatalo for (ApiInfoCatalog apiInfoCatalog: apiInfoCatalogMap.values()) { Map strictURLToMethods = apiInfoCatalog.getStrictURLToMethods(); - Map templateURLToMethods = apiInfoCatalog.getTemplateURLToMethods(); + Map templateURLToMethods = new HashMap<>(); + + Set mergedUrls = MergedUrlsDao.instance.getMergedUrls(); + for(Map.Entry templateURLToMethodEntry : apiInfoCatalog.getTemplateURLToMethods().entrySet()) { + ApiInfoKey apiInfoKey = templateURLToMethodEntry.getValue().getApiInfo().getId(); + if(!mergedUrls.contains(new MergedUrls(apiInfoKey.getUrl(), apiInfoKey.getMethod().name(), apiInfoKey.getApiCollectionId()))) { + templateURLToMethods.put(templateURLToMethodEntry.getKey(), templateURLToMethodEntry.getValue()); + } + } List policyCatalogList = new ArrayList<>(); policyCatalogList.addAll(strictURLToMethods.values()); diff --git a/apps/dashboard/src/test/java/com/akto/action/TestHarAction.java b/apps/dashboard/src/test/java/com/akto/action/TestHarAction.java index 67f2a9bc8c..eeb118d92b 100644 --- a/apps/dashboard/src/test/java/com/akto/action/TestHarAction.java +++ b/apps/dashboard/src/test/java/com/akto/action/TestHarAction.java @@ -1,36 +1,69 @@ package com.akto.action; -import com.akto.DaoInit; import com.akto.MongoBasedTest; +import com.akto.action.observe.InventoryAction; import com.akto.analyser.ResourceAnalyser; import com.akto.dao.AccountSettingsDao; import com.akto.dao.ApiCollectionsDao; +import com.akto.dao.ApiInfoDao; import com.akto.dao.SingleTypeInfoDao; import com.akto.dao.context.Context; -import com.akto.dto.AccountSettings; -import com.akto.dto.ApiCollection; -import com.akto.dto.User; -import com.akto.dto.type.SingleTypeInfo; +import com.akto.dto.*; +import com.akto.dto.type.RequestTemplate; +import com.akto.dto.type.URLStatic; +import com.akto.dto.type.URLTemplate; import com.akto.listener.RuntimeListener; import com.akto.parsers.HttpCallParser; -import com.akto.runtime.policies.AktoPolicyNew; import com.akto.utils.AccountHTTPCallParserAktoPolicyInfo; -import com.mongodb.ConnectionString; +import com.mongodb.client.model.Filters; import org.bson.conversions.Bson; -import org.checkerframework.checker.units.qual.A; -import org.junit.Ignore; import org.junit.Test; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; +import java.util.*; import static org.junit.Assert.assertEquals; public class TestHarAction extends MongoBasedTest{ + @Test + public void testDemergeAfterReuploadingApis() throws Exception { + HttpCallParser httpCallParser = new HttpCallParser("",0,0,0, true); + + String payload = "{\"method\":\"GET\",\"requestPayload\":\"{\\\"photoUrls\\\":[\\\"string\\\"],\\\"name\\\":\\\"doggie\\\",\\\"id\\\":0,\\\"category\\\":{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"},\\\"tags\\\":[{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"}],\\\"status\\\":\\\"available\\\"}\",\"responsePayload\":\"{\\\"id\\\":9223372036854775807,\\\"category\\\":{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"},\\\"name\\\":\\\"doggie\\\",\\\"photoUrls\\\":[\\\"string\\\"],\\\"tags\\\":[{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"}],\\\"status\\\":\\\"available\\\"}\",\"ip\":\"null\",\"source\":\"MIRRORING\",\"type\":\"HTTP/2\",\"akto_vxlan_id\":\"1661807253\",\"path\":\"https://juice-shop.herokuapp.com/api/Deliverys/ec6d5f9d-94a7-4096-bcf1-7a0818bba867\",\"requestHeaders\":\"{\\\"Origin\\\":\\\"https://petstore.swagger.io\\\",\\\"Accept\\\":\\\"application/json\\\",\\\"User-Agent\\\":\\\"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:95.0) Gecko/20100101 Firefox/95.0\\\",\\\"Referer\\\":\\\"https://petstore.swagger.io/\\\",\\\"Connection\\\":\\\"keep-alive\\\",\\\"Sec-Fetch-Dest\\\":\\\"empty\\\",\\\"Sec-Fetch-Site\\\":\\\"same-origin\\\",\\\"Host\\\":\\\"petstore.swagger.io\\\",\\\"Accept-Encoding\\\":\\\"gzip, deflate, br\\\",\\\"Sec-Fetch-Mode\\\":\\\"cors\\\",\\\"TE\\\":\\\"trailers\\\",\\\"Accept-Language\\\":\\\"en-US,en;q=0.5\\\",\\\"Content-Length\\\":\\\"215\\\",\\\"Content-Type\\\":\\\"application/json\\\"}\",\"responseHeaders\":\"{\\\"date\\\":\\\"Tue, 04 Jan 2022 20:11:58 GMT\\\",\\\"access-control-allow-origin\\\":\\\"*\\\",\\\"server\\\":\\\"Jetty(9.2.9.v20150224)\\\",\\\"access-control-allow-headers\\\":\\\"Content-Type, api_key, Authorization\\\",\\\"X-Firefox-Spdy\\\":\\\"h2\\\",\\\"content-type\\\":\\\"application/json\\\",\\\"access-control-allow-methods\\\":\\\"GET, POST, DELETE, PUT\\\"}\",\"time\":\"1641327118\",\"contentType\":\"application/json\",\"akto_account_id\":\"1000000\",\"statusCode\":\"200\",\"status\":\"OK\"}"; + HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(payload); + httpCallParser.syncFunction(Collections.singletonList(httpResponseParams),true, true, null); + + List apiInfoList = ApiInfoDao.instance.findAll(Filters.empty()); + assertEquals("https://juice-shop.herokuapp.com/api/Deliverys/STRING", apiInfoList.get(0).getId().getUrl()); + + InventoryAction action = new InventoryAction(); + action.setUrl("https://juice-shop.herokuapp.com/api/Deliverys/STRING"); + action.setMethod("GET"); + action.setApiCollectionId(335433302); + action.deMergeApi(); + + Map strictURLToMethods = httpCallParser.apiCatalogSync.dbState.get(335433302).getStrictURLToMethods(); + Map templateURLToMethods = httpCallParser.apiCatalogSync.dbState.get(335433302).getTemplateURLToMethods(); + + assertEquals(0, strictURLToMethods.size()); + assertEquals(1, templateURLToMethods.size()); + + List apiInfoList1 = ApiInfoDao.instance.findAll(Filters.empty()); + assertEquals("https://juice-shop.herokuapp.com/api/Deliverys/ec6d5f9d-94a7-4096-bcf1-7a0818bba867", apiInfoList1.get(0).getId().getUrl()); + + httpCallParser.syncFunction(Collections.singletonList(httpResponseParams),true, true, null); + + Map strictURLToMethods1 = httpCallParser.apiCatalogSync.dbState.get(335433302).getStrictURLToMethods(); + Map templateURLToMethods1 = httpCallParser.apiCatalogSync.dbState.get(335433302).getTemplateURLToMethods(); + + assertEquals(1, strictURLToMethods1.size()); + assertEquals(0, templateURLToMethods1.size()); + + List apiInfoList2 = ApiInfoDao.instance.findAll(Filters.empty()); + assertEquals("https://juice-shop.herokuapp.com/api/Deliverys/ec6d5f9d-94a7-4096-bcf1-7a0818bba867", apiInfoList2.get(0).getId().getUrl()); + } + @Test public void testHeaderFilter() throws IOException { diff --git a/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java b/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java index c1ba13af1b..c0b529dd37 100644 --- a/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java +++ b/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java @@ -778,9 +778,12 @@ public static String[] tokenize(String url) { } Map convertToMap(List l) { + Set mergedUrls = MergedUrlsDao.instance.getMergedUrls(); Map ret = new HashMap<>(); for(SingleTypeInfo e: l) { - ret.put(e.composeKey(), e); + if(!mergedUrls.contains(new MergedUrls(e.getUrl(), e.getMethod(), e.getApiCollectionId()))) { + ret.put(e.composeKey(), e); + } } return ret; diff --git a/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/policies/AktoPolicyNew.java b/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/policies/AktoPolicyNew.java index ed0e9ac083..a13d068343 100644 --- a/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/policies/AktoPolicyNew.java +++ b/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/policies/AktoPolicyNew.java @@ -2,8 +2,10 @@ import com.akto.dao.*; import com.akto.dao.context.Context; +import com.akto.dao.filter.MergedUrlsDao; import com.akto.dto.*; import com.akto.dto.ApiInfo.ApiInfoKey; +import com.akto.dto.filter.MergedUrls; import com.akto.dto.runtime_filters.RuntimeFilter; import com.akto.dto.type.APICatalog; import com.akto.dto.type.SingleTypeInfo; @@ -269,7 +271,15 @@ public static List getUpdates(Map apiInfoCatal for (ApiInfoCatalog apiInfoCatalog: apiInfoCatalogMap.values()) { Map strictURLToMethods = apiInfoCatalog.getStrictURLToMethods(); - Map templateURLToMethods = apiInfoCatalog.getTemplateURLToMethods(); + Map templateURLToMethods = new HashMap<>(); + + Set mergedUrls = MergedUrlsDao.instance.getMergedUrls(); + for(Map.Entry templateURLToMethodEntry : apiInfoCatalog.getTemplateURLToMethods().entrySet()) { + ApiInfoKey apiInfoKey = templateURLToMethodEntry.getValue().getApiInfo().getId(); + if(!mergedUrls.contains(new MergedUrls(apiInfoKey.getUrl(), apiInfoKey.getMethod().name(), apiInfoKey.getApiCollectionId()))) { + templateURLToMethods.put(templateURLToMethodEntry.getKey(), templateURLToMethodEntry.getValue()); + } + } List policyCatalogList = new ArrayList<>(); policyCatalogList.addAll(strictURLToMethods.values());