diff --git a/deepfence_agent/Dockerfile.cloud-agent b/deepfence_agent/Dockerfile.cloud-agent index 7ee0397a3e..22baac87ad 100644 --- a/deepfence_agent/Dockerfile.cloud-agent +++ b/deepfence_agent/Dockerfile.cloud-agent @@ -29,7 +29,11 @@ WORKDIR /opt/steampipe USER deepfence -ENV DF_INSTALL_DIR=/home/deepfence +ENV DF_INSTALL_DIR=/home/deepfence \ + STEAMPIPE_AWS_PLUGIN_VERSION=0.118.1 \ + STEAMPIPE_GCP_PLUGIN_VERSION=0.43.0 \ + STEAMPIPE_AZURE_PLUGIN_VERSION=0.49.0 \ + STEAMPIPE_AZURE_AD_PLUGIN_VERSION=0.12.0 COPY supervisord-cloud.conf /home/deepfence/supervisord.conf COPY --from=steampipe /usr/local/bin/steampipe /usr/local/bin/steampipe @@ -37,7 +41,7 @@ COPY --from=steampipe /usr/local/bin/steampipe /usr/local/bin/steampipe RUN steampipe service start \ && steampipe plugin install steampipe \ # plugin version should be in sync with Deepfence fork https://github.com/deepfence/steampipe-plugin-aws - && steampipe plugin install aws@0.118.1 gcp@0.43.0 azure@0.49.0 azuread@0.12.0 \ + && steampipe plugin install aws@${STEAMPIPE_AWS_PLUGIN_VERSION} gcp@${STEAMPIPE_GCP_PLUGIN_VERSION} azure@${STEAMPIPE_AZURE_PLUGIN_VERSION} azuread@${STEAMPIPE_AZURE_AD_PLUGIN_VERSION} \ && git clone https://github.com/turbot/steampipe-mod-aws-compliance.git --branch v0.79 --depth 1 \ && git clone https://github.com/turbot/steampipe-mod-gcp-compliance.git --branch v0.21 --depth 1 \ && git clone https://github.com/turbot/steampipe-mod-azure-compliance.git --branch v0.35 --depth 1 \ @@ -56,10 +60,10 @@ ENV PUBLISH_CLOUD_RESOURCES_INTERVAL_MINUTES=5 \ EXPOSE 8080 -COPY --from=steampipe /usr/local/bin/steampipe-plugin-aws.plugin /home/deepfence/.steampipe/plugins/hub.steampipe.io/plugins/turbot/aws@latest/steampipe-plugin-aws.plugin -COPY --from=steampipe /usr/local/bin/steampipe-plugin-gcp.plugin /home/deepfence/.steampipe/plugins/hub.steampipe.io/plugins/turbot/gcp@latest/steampipe-plugin-gcp.plugin -COPY --from=steampipe /usr/local/bin/steampipe-plugin-azure.plugin /home/deepfence/.steampipe/plugins/hub.steampipe.io/plugins/turbot/azure@latest/steampipe-plugin-azure.plugin -COPY --from=steampipe /usr/local/bin/steampipe-plugin-azuread.plugin /home/deepfence/.steampipe/plugins/hub.steampipe.io/plugins/turbot/azuread@latest/steampipe-plugin-azuread.plugin +COPY --from=steampipe /usr/local/bin/steampipe-plugin-aws.plugin /home/deepfence/.steampipe/plugins/hub.steampipe.io/plugins/turbot/aws@${STEAMPIPE_AWS_PLUGIN_VERSION}/steampipe-plugin-aws.plugin +COPY --from=steampipe /usr/local/bin/steampipe-plugin-gcp.plugin /home/deepfence/.steampipe/plugins/hub.steampipe.io/plugins/turbot/gcp@${STEAMPIPE_GCP_PLUGIN_VERSION}/steampipe-plugin-gcp.plugin +COPY --from=steampipe /usr/local/bin/steampipe-plugin-azure.plugin /home/deepfence/.steampipe/plugins/hub.steampipe.io/plugins/turbot/azure@${STEAMPIPE_AZURE_PLUGIN_VERSION}/steampipe-plugin-azure.plugin +COPY --from=steampipe /usr/local/bin/steampipe-plugin-azuread.plugin /home/deepfence/.steampipe/plugins/hub.steampipe.io/plugins/turbot/azuread@${STEAMPIPE_AZURE_AD_PLUGIN_VERSION}/steampipe-plugin-azuread.plugin COPY plugins/cloud-scanner/cloud_scanner /home/deepfence/bin/cloud_scanner diff --git a/deepfence_agent/plugins/YaraHunter b/deepfence_agent/plugins/YaraHunter index 8e4d161ce8..3cef40ff4f 160000 --- a/deepfence_agent/plugins/YaraHunter +++ b/deepfence_agent/plugins/YaraHunter @@ -1 +1 @@ -Subproject commit 8e4d161ce812f31a0afd6d1a0d4f8c5440845f47 +Subproject commit 3cef40ff4f7408bca29613b304e277451bd3da4b diff --git a/deepfence_agent/plugins/cloud-scanner b/deepfence_agent/plugins/cloud-scanner index c25aedcfdb..402d361b8f 160000 --- a/deepfence_agent/plugins/cloud-scanner +++ b/deepfence_agent/plugins/cloud-scanner @@ -1 +1 @@ -Subproject commit c25aedcfdbc09a9deae9f40cdea9367a6b6ca658 +Subproject commit 402d361b8f543cc008ceb31bf44cc0fa6fcead16 diff --git a/deepfence_agent/plugins/yara-rules b/deepfence_agent/plugins/yara-rules index 8217b51893..bc374a2ad6 160000 --- a/deepfence_agent/plugins/yara-rules +++ b/deepfence_agent/plugins/yara-rules @@ -1 +1 @@ -Subproject commit 8217b518934b556ee7f56e6e5fc3e05be8c8d9fa +Subproject commit bc374a2ad6b15aa70894159b670bef03f2d482e7 diff --git a/deepfence_bootstrapper/router/cloud_scanner.go b/deepfence_bootstrapper/router/cloud_scanner.go index a7166d927c..f0455c1000 100644 --- a/deepfence_bootstrapper/router/cloud_scanner.go +++ b/deepfence_bootstrapper/router/cloud_scanner.go @@ -89,7 +89,7 @@ func RefreshResources(req ctl.RefreshResourcesRequest) error { return nil } -func GetCloudScannerJobCount() int32 { +func GetCloudScannerJobCount(action ctl.ActionID) int32 { conn, err := net.Dial("unix", CloudScannerSocketPath) if err != nil { log.Error().Err(err).Msgf("GetCloudScannerJobCount: error in creating cloud compliance scanner client with socket %s", CloudScannerSocketPath) @@ -98,7 +98,7 @@ func GetCloudScannerJobCount() int32 { defer conn.Close() jobCountReq := map[string]interface{}{ - "action": ctl.CloudScannerJobCount, + "action": action, } jobCountReqBytes, err := json.Marshal(jobCountReq) if err != nil { @@ -128,49 +128,3 @@ func GetCloudScannerJobCount() int32 { return jobCount } } - -func GetCloudNodeID() (string, error) { - cloudNodeID := "" - conn, err := net.Dial("unix", CloudScannerSocketPath) - if err != nil { - log.Error().Err(err).Msgf("Error creating cloud scanner client with socket %s", CloudScannerSocketPath) - return cloudNodeID, err - } - defer conn.Close() - reqMap := make(map[string]interface{}) - reqMap["GetCloudNodeID"] = true - cloudNodeIDReq := map[string]interface{}{ - "args": reqMap, - } - - cloudNodeIDReqBytes, err := json.Marshal(cloudNodeIDReq) - if err != nil { - log.Error().Err(err).Msg("Error in converting request into valid json") - return cloudNodeID, err - } - - _, err = conn.Write(cloudNodeIDReqBytes) - if err != nil { - log.Error().Err(err).Msgf("Error in writing data to unix socket %s", CloudScannerSocketPath) - return cloudNodeID, err - } - - responseTimeout := 10 * time.Second - deadline := time.Now().Add(responseTimeout) - buf := make([]byte, 1024) - for { - conn.SetReadDeadline(deadline) - n, err := conn.Read(buf[:]) - if err != nil { - log.Error().Err(err).Msg("Error in read") - return cloudNodeID, err - } - - count, err := fmt.Sscan(string(buf[0:n]), &cloudNodeID) - if err != nil || count != 1 { - return cloudNodeID, err - } - break - } - return cloudNodeID, err -} diff --git a/deepfence_bootstrapper/router/openapi_client_controls.go b/deepfence_bootstrapper/router/openapi_client_controls.go index 9832d50f9f..235074e4af 100644 --- a/deepfence_bootstrapper/router/openapi_client_controls.go +++ b/deepfence_bootstrapper/router/openapi_client_controls.go @@ -119,25 +119,29 @@ func (ct *OpenapiClient) StartControlsWatching(nodeID string, const ( MaxAgentWorkload = 2 - MaxCloudAgentWorkload = 1 + MaxCloudAgentWorkload = 2 ) func GetScannersWorkloads(nodeType string) int32 { - res := int32(0) - var secret, malware, vuln, cloud int32 if nodeType == ctl.CLOUD_AGENT { - cloud = GetCloudScannerJobCount() + var cloudPostureScan, cloudResourceRefreshCount int32 + + cloudPostureScan = GetCloudScannerJobCount(ctl.CloudScannerJobCount) + cloudResourceRefreshCount = GetCloudScannerJobCount(ctl.CloudScannerResourceRefreshCount) + + log.Info().Msgf("workloads = cloud posture: %d, cloud resource refresh: %d", cloudPostureScan, cloudResourceRefreshCount) + return cloudPostureScan + cloudResourceRefreshCount } else { + var secret, malware, vuln int32 + secret = GetSecretScannerJobCount() malware = GetMalwareScannerJobCount() vuln = GetPackageScannerJobCount() - } - //TODO: Add more scanners workload - log.Info().Msgf("workloads = vuln: %d, secret: %d, malware: %d, cloud: %d", - vuln, secret, malware, cloud) - res = secret + malware + vuln + cloud - return res + //TODO: Add more scanners workload + log.Info().Msgf("workloads = vuln: %d, secret: %d, malware: %d", vuln, secret, malware) + return secret + malware + vuln + } } var upgrade atomic.Bool diff --git a/deepfence_frontend/apps/dashboard/api-spec.json b/deepfence_frontend/apps/dashboard/api-spec.json index 503677fd85..b1bd59b3d3 100644 --- a/deepfence_frontend/apps/dashboard/api-spec.json +++ b/deepfence_frontend/apps/dashboard/api-spec.json @@ -13616,9 +13616,11 @@ "node_id": { "type": "string" }, "node_name": { "type": "string" }, "refresh_message": { "type": "string" }, - "refresh_status": { - "enum": ["STARTING", "IN_PROGRESS", "ERROR", "COMPLETE"], - "type": "string" + "refresh_status": { "type": "string" }, + "refresh_status_map": { + "type": "object", + "additionalProperties": { "type": "integer" }, + "nullable": true }, "scan_status_map": { "type": "object", @@ -14587,7 +14589,8 @@ "config": { "type": "object", "additionalProperties": {}, "nullable": true }, "filters": { "$ref": "#/components/schemas/ModelIntegrationFilters" }, "integration_type": { "type": "string" }, - "notification_type": { "type": "string" } + "notification_type": { "type": "string" }, + "send_summary": { "type": "boolean" } } }, "ModelIntegrationFilters": { @@ -14625,7 +14628,8 @@ "filters": { "$ref": "#/components/schemas/ModelIntegrationFilters" }, "id": { "type": "integer" }, "integration_type": { "type": "string" }, - "notification_type": { "type": "string" } + "notification_type": { "type": "string" }, + "send_summary": { "type": "boolean" } } }, "ModelInviteUserRequest": { diff --git a/deepfence_frontend/apps/dashboard/src/api/generated/models/ModelCloudNodeAccountInfo.ts b/deepfence_frontend/apps/dashboard/src/api/generated/models/ModelCloudNodeAccountInfo.ts index 29775ee60e..e493165119 100644 --- a/deepfence_frontend/apps/dashboard/src/api/generated/models/ModelCloudNodeAccountInfo.ts +++ b/deepfence_frontend/apps/dashboard/src/api/generated/models/ModelCloudNodeAccountInfo.ts @@ -84,7 +84,13 @@ export interface ModelCloudNodeAccountInfo { * @type {string} * @memberof ModelCloudNodeAccountInfo */ - refresh_status?: ModelCloudNodeAccountInfoRefreshStatusEnum; + refresh_status?: string; + /** + * + * @type {{ [key: string]: number; }} + * @memberof ModelCloudNodeAccountInfo + */ + refresh_status_map?: { [key: string]: number; } | null; /** * * @type {{ [key: string]: number; }} @@ -113,17 +119,6 @@ export const ModelCloudNodeAccountInfoCloudProviderEnum = { } as const; export type ModelCloudNodeAccountInfoCloudProviderEnum = typeof ModelCloudNodeAccountInfoCloudProviderEnum[keyof typeof ModelCloudNodeAccountInfoCloudProviderEnum]; -/** - * @export - */ -export const ModelCloudNodeAccountInfoRefreshStatusEnum = { - Starting: 'STARTING', - InProgress: 'IN_PROGRESS', - Error: 'ERROR', - Complete: 'COMPLETE' -} as const; -export type ModelCloudNodeAccountInfoRefreshStatusEnum = typeof ModelCloudNodeAccountInfoRefreshStatusEnum[keyof typeof ModelCloudNodeAccountInfoRefreshStatusEnum]; - /** * Check if a given object implements the ModelCloudNodeAccountInfo interface. @@ -155,6 +150,7 @@ export function ModelCloudNodeAccountInfoFromJSONTyped(json: any, ignoreDiscrimi 'node_name': !exists(json, 'node_name') ? undefined : json['node_name'], 'refresh_message': !exists(json, 'refresh_message') ? undefined : json['refresh_message'], 'refresh_status': !exists(json, 'refresh_status') ? undefined : json['refresh_status'], + 'refresh_status_map': !exists(json, 'refresh_status_map') ? undefined : json['refresh_status_map'], 'scan_status_map': !exists(json, 'scan_status_map') ? undefined : json['scan_status_map'], 'version': !exists(json, 'version') ? undefined : json['version'], }; @@ -180,6 +176,7 @@ export function ModelCloudNodeAccountInfoToJSON(value?: ModelCloudNodeAccountInf 'node_name': value.node_name, 'refresh_message': value.refresh_message, 'refresh_status': value.refresh_status, + 'refresh_status_map': value.refresh_status_map, 'scan_status_map': value.scan_status_map, 'version': value.version, }; diff --git a/deepfence_frontend/apps/dashboard/src/api/generated/models/ModelIntegrationAddReq.ts b/deepfence_frontend/apps/dashboard/src/api/generated/models/ModelIntegrationAddReq.ts index f5211dd456..7a68b9abfa 100644 --- a/deepfence_frontend/apps/dashboard/src/api/generated/models/ModelIntegrationAddReq.ts +++ b/deepfence_frontend/apps/dashboard/src/api/generated/models/ModelIntegrationAddReq.ts @@ -50,6 +50,12 @@ export interface ModelIntegrationAddReq { * @memberof ModelIntegrationAddReq */ notification_type: string; + /** + * + * @type {boolean} + * @memberof ModelIntegrationAddReq + */ + send_summary?: boolean; } /** @@ -77,6 +83,7 @@ export function ModelIntegrationAddReqFromJSONTyped(json: any, ignoreDiscriminat 'filters': !exists(json, 'filters') ? undefined : ModelIntegrationFiltersFromJSON(json['filters']), 'integration_type': json['integration_type'], 'notification_type': json['notification_type'], + 'send_summary': !exists(json, 'send_summary') ? undefined : json['send_summary'], }; } @@ -93,6 +100,7 @@ export function ModelIntegrationAddReqToJSON(value?: ModelIntegrationAddReq | nu 'filters': ModelIntegrationFiltersToJSON(value.filters), 'integration_type': value.integration_type, 'notification_type': value.notification_type, + 'send_summary': value.send_summary, }; } diff --git a/deepfence_frontend/apps/dashboard/src/api/generated/models/ModelIntegrationUpdateReq.ts b/deepfence_frontend/apps/dashboard/src/api/generated/models/ModelIntegrationUpdateReq.ts index d6406f84a5..c000619da9 100644 --- a/deepfence_frontend/apps/dashboard/src/api/generated/models/ModelIntegrationUpdateReq.ts +++ b/deepfence_frontend/apps/dashboard/src/api/generated/models/ModelIntegrationUpdateReq.ts @@ -56,6 +56,12 @@ export interface ModelIntegrationUpdateReq { * @memberof ModelIntegrationUpdateReq */ notification_type?: string; + /** + * + * @type {boolean} + * @memberof ModelIntegrationUpdateReq + */ + send_summary?: boolean; } /** @@ -82,6 +88,7 @@ export function ModelIntegrationUpdateReqFromJSONTyped(json: any, ignoreDiscrimi 'id': !exists(json, 'id') ? undefined : json['id'], 'integration_type': !exists(json, 'integration_type') ? undefined : json['integration_type'], 'notification_type': !exists(json, 'notification_type') ? undefined : json['notification_type'], + 'send_summary': !exists(json, 'send_summary') ? undefined : json['send_summary'], }; } @@ -99,6 +106,7 @@ export function ModelIntegrationUpdateReqToJSON(value?: ModelIntegrationUpdateRe 'id': value.id, 'integration_type': value.integration_type, 'notification_type': value.notification_type, + 'send_summary': value.send_summary, }; } diff --git a/deepfence_frontend/apps/dashboard/src/components/ScanStatusBadge.tsx b/deepfence_frontend/apps/dashboard/src/components/ScanStatusBadge.tsx index 690ab2128e..b9ec63aef9 100644 --- a/deepfence_frontend/apps/dashboard/src/components/ScanStatusBadge.tsx +++ b/deepfence_frontend/apps/dashboard/src/components/ScanStatusBadge.tsx @@ -1,6 +1,6 @@ import { capitalize } from 'lodash-es'; import { cn } from 'tailwind-preset'; -import { CircleSpinner } from 'ui-components'; +import { CircleSpinner, Tooltip } from 'ui-components'; import { ErrorIcon, @@ -22,10 +22,12 @@ export const ScanStatusBadge = ({ status, className, justIcon = false, + errorMessage, }: { status: string; className?: string; justIcon?: boolean; + errorMessage?: string; }) => { const wrapperClassName = cn( 'flex items-center gap-1.5 text-text-text-and-icon text-p4a', @@ -49,10 +51,23 @@ export const ScanStatusBadge = ({ } else if (isScanFailed(status)) { return (
- - - - {!justIcon ? : null} + {errorMessage ? ( + +
+ + + + {!justIcon ? : null} +
+ + ) : ( + <> + + + + {!justIcon ? : null} + + )}
); } else if (isNeverScanned(status)) { diff --git a/deepfence_frontend/apps/dashboard/src/features/integrations/components/integration-form/NotificationTypeField.tsx b/deepfence_frontend/apps/dashboard/src/features/integrations/components/integration-form/NotificationTypeField.tsx index b77917ca28..9ae75b700e 100644 --- a/deepfence_frontend/apps/dashboard/src/features/integrations/components/integration-form/NotificationTypeField.tsx +++ b/deepfence_frontend/apps/dashboard/src/features/integrations/components/integration-form/NotificationTypeField.tsx @@ -1,14 +1,16 @@ import { useState } from 'react'; import { useParams } from 'react-router-dom'; -import { Listbox, ListboxOption } from 'ui-components'; +import { Checkbox, Listbox, ListboxOption, Tooltip } from 'ui-components'; import { ModelIntegrationListResp } from '@/api/generated'; +import { InfoStandardIcon } from '@/components/icons/common/InfoStandard'; import { ScanTypeEnum } from '@/types/common'; import { AdvancedFilters } from './AdvancedFilter'; import { FieldSelection } from './FieldSelection'; import { TextInputType } from './TextInputType'; import { + canSendScanSummary, getNotificationPrettyName, IntegrationType, isCloudComplianceNotification, @@ -18,6 +20,24 @@ import { isVulnerabilityNotification, } from './utils'; +const SendScanSummaryCheckbox = ({ sendSummaryOnly }: { sendSummaryOnly: boolean }) => { + const [checked, setChecked] = useState(sendSummaryOnly); + return ( +
+ setChecked(check)} + /> + +
+ +
+ +
+ ); +}; export const NotificationTypeField = ({ fieldErrors, defaultNotificationType, @@ -80,6 +100,10 @@ export const NotificationTypeField = ({ ) : null} */} + {canSendScanSummary(notificationType, integrationType) ? ( + + ) : null} + {isCloudComplianceNotification(notificationType) && integrationType !== IntegrationType.s3 && ( { } return notificationType; }; + +export const canSendScanSummary = (notificationType: string, integrationType: string) => { + return ( + ((integrationType === IntegrationType.slack || + integrationType === IntegrationType.microsoftTeams) && + ['Secret', 'Vulnerability', 'Malware'].includes(notificationType)) || + isComplianceNotification(notificationType) || + isCloudComplianceNotification(notificationType) + ); +}; diff --git a/deepfence_frontend/apps/dashboard/src/features/integrations/components/report-form/AdvanceFilter.tsx b/deepfence_frontend/apps/dashboard/src/features/integrations/components/report-form/AdvanceFilter.tsx index 7639d34bce..7887d4ab84 100644 --- a/deepfence_frontend/apps/dashboard/src/features/integrations/components/report-form/AdvanceFilter.tsx +++ b/deepfence_frontend/apps/dashboard/src/features/integrations/components/report-form/AdvanceFilter.tsx @@ -1,4 +1,3 @@ -import { upperCase } from 'lodash-es'; import { useMemo, useState } from 'react'; import { Listbox, ListboxOption } from 'ui-components'; diff --git a/deepfence_frontend/apps/dashboard/src/features/integrations/pages/IntegrationAdd.tsx b/deepfence_frontend/apps/dashboard/src/features/integrations/pages/IntegrationAdd.tsx index d578d8bf5d..91b69f8c9a 100644 --- a/deepfence_frontend/apps/dashboard/src/features/integrations/pages/IntegrationAdd.tsx +++ b/deepfence_frontend/apps/dashboard/src/features/integrations/pages/IntegrationAdd.tsx @@ -316,6 +316,7 @@ const action = async ({ request, params }: ActionFunctionArgs): Promise { - return ( -
-
Error
- {errorMessage ? ( - -
- -
- - ) : ( -
- -
- )} -
- ); -}; diff --git a/deepfence_frontend/apps/dashboard/src/features/postures/pages/Accounts.tsx b/deepfence_frontend/apps/dashboard/src/features/postures/pages/Accounts.tsx index dfb8315fe0..210298d737 100644 --- a/deepfence_frontend/apps/dashboard/src/features/postures/pages/Accounts.tsx +++ b/deepfence_frontend/apps/dashboard/src/features/postures/pages/Accounts.tsx @@ -1,6 +1,6 @@ import { useSuspenseQuery } from '@suspensive/react-query'; import { useIsFetching } from '@tanstack/react-query'; -import { capitalize, startCase } from 'lodash-es'; +import { capitalize } from 'lodash-es'; import { Suspense, useCallback, useEffect, useMemo, useState } from 'react'; import { ActionFunctionArgs, @@ -10,6 +10,7 @@ import { useSearchParams, } from 'react-router-dom'; import { toast } from 'sonner'; +import { cn } from 'tailwind-preset'; import { Badge, Breadcrumb, @@ -65,7 +66,6 @@ import { getColorForCompliancePercent } from '@/constants/charts'; import { BreadcrumbWrapper } from '@/features/common/BreadcrumbWrapper'; import { useDownloadScan } from '@/features/common/data-component/downloadScanAction'; import { FilterWrapper } from '@/features/common/FilterWrapper'; -import { RefreshAccountStatusError } from '@/features/postures/components/RefreshAccountStatusError'; import { providersToNameMapping } from '@/features/postures/pages/Posture'; import { getDeleteConfirmationDisplayName, @@ -915,6 +915,7 @@ const ActionDropdown = ({ if (!scanId || !nodeType) return; onTableAction(row, ActionEnumType.DELETE_SCAN); }} + color="error" > Delete latest scan @@ -1338,23 +1339,17 @@ const AccountTable = ({ cell: (info) => { if (nodeType?.endsWith?.('_org')) { const data = info.row.original.scan_status_map ?? {}; - const keys = Object.keys(data); - const statuses = Object.keys(data).map((current, index) => { + const statuses = Object.keys(data).map((current) => { return ( - <> -
- - {data[current]} - - - {index < keys.length - 1 ? ( -
- ) : null} -
- +
+ + {data[current]} + + +
); }); - return
{statuses}
; + return
{statuses}
; } else { const value = info.getValue(); return ; @@ -1367,7 +1362,11 @@ const AccountTable = ({ ...columnWidth.active, header: () => 'Active', cell: (info) => { - return info.getValue() ? 'Yes' : 'No'; + return ( + + {info.getValue() ? 'Yes' : 'No'} + + ); }, }), ]; @@ -1380,7 +1379,9 @@ const AccountTable = ({ ...columnWidth.account_name, header: () => 'Name', cell: (info) => { - return ; + return ( + + ); }, }), ); @@ -1391,14 +1392,30 @@ const AccountTable = ({ ...columnWidth.refresh_status, header: () => 'Refresh status', cell: (info) => { - if (isRefreshAccountFailed(info.getValue())) { + if (nodeType?.endsWith?.('_org')) { + const data = info.row.original.refresh_status_map ?? {}; + const statuses = Object.keys(data).map((current) => { + return ( +
+ + {data[current]} + + +
+ ); + }); + return
{statuses}
; + } else { return ( - ); } - return ; }, }), ); diff --git a/deepfence_frontend/apps/dashboard/src/features/postures/utils/index.ts b/deepfence_frontend/apps/dashboard/src/features/postures/utils/index.ts index 0e9f05f071..fe67bab5bc 100644 --- a/deepfence_frontend/apps/dashboard/src/features/postures/utils/index.ts +++ b/deepfence_frontend/apps/dashboard/src/features/postures/utils/index.ts @@ -1,8 +1,8 @@ import { ModelCloudComplianceStatusEnum, - ModelCloudNodeAccountInfoRefreshStatusEnum, ModelCloudNodeAccountsListReqCloudProviderEnum, ModelComplianceStatusEnum, + ModelScanInfoStatusEnum, } from '@/api/generated'; export const isCloudNonOrgNode = (nodeType?: string) => { @@ -122,7 +122,7 @@ export function getDeleteConfirmationDisplayName( } export const isRefreshAccountFailed = (status: string): boolean => { - if (status?.length && ModelCloudNodeAccountInfoRefreshStatusEnum.Error === status) { + if (status?.length && ModelScanInfoStatusEnum.Error === status) { return true; } return false; diff --git a/deepfence_frontend/apps/dashboard/src/features/topology/data-components/tables/HostsTable.tsx b/deepfence_frontend/apps/dashboard/src/features/topology/data-components/tables/HostsTable.tsx index f25d090274..08e394d202 100644 --- a/deepfence_frontend/apps/dashboard/src/features/topology/data-components/tables/HostsTable.tsx +++ b/deepfence_frontend/apps/dashboard/src/features/topology/data-components/tables/HostsTable.tsx @@ -1,6 +1,8 @@ import { useSuspenseQuery } from '@suspensive/react-query'; +import { startCase } from 'lodash-es'; import { Suspense, useEffect, useMemo, useState } from 'react'; import { useSearchParams } from 'react-router-dom'; +import { cn } from 'tailwind-preset'; import { Badge, Button, @@ -546,7 +548,7 @@ function Filters() { }} getDisplayValue={() => FILTER_SEARCHPARAMS['agentRunning']} > - {['On', 'Off'] + {['Yes', 'No'] .filter((item) => { if (!agentRunningSearchText.length) return true; return item.toLowerCase().includes(agentRunningSearchText.toLowerCase()); @@ -639,7 +641,7 @@ function useSearchHostsWithPagination() { order: getOrderFromSearchParams(searchParams), agentRunning: searchParams .getAll('agentRunning') - .map((value) => (value === 'On' ? true : false)), + .map((value) => (value === 'Yes' ? true : false)), cloudAccounts: searchParams.getAll('cloudAccounts'), clusterIds: searchParams.getAll('clusters'), hosts: searchParams.getAll('hosts'), @@ -723,8 +725,8 @@ const DataTable = ({ }, header: () => 'Name', minSize: 300, - size: 400, - maxSize: 600, + size: 320, + maxSize: 360, }), columnHelper.accessor('vulnerability_scan_status', { cell: (info) => { @@ -767,20 +769,27 @@ const DataTable = ({ if (!info.getValue()) { return
; } - return ; + return ; }, - header: () => OS, + header: () => 'OS', minSize: 50, size: 60, maxSize: 120, }), columnHelper.accessor('agent_running', { cell: (info) => { - return ; + return ( + + ); }, header: () => , - minSize: 50, - size: 60, + minSize: 80, + size: 100, maxSize: 120, }), columnHelper.accessor('version', { @@ -827,6 +836,9 @@ const DataTable = ({ ); } } + if (!info.getValue()) { + return
; + } return ; }, header: () => , diff --git a/deepfence_frontend/apps/dashboard/src/features/topology/data-components/tables/KubernetesTable.tsx b/deepfence_frontend/apps/dashboard/src/features/topology/data-components/tables/KubernetesTable.tsx index d78cabb589..a207676d3c 100644 --- a/deepfence_frontend/apps/dashboard/src/features/topology/data-components/tables/KubernetesTable.tsx +++ b/deepfence_frontend/apps/dashboard/src/features/topology/data-components/tables/KubernetesTable.tsx @@ -1,7 +1,7 @@ import { useSuspenseQuery } from '@suspensive/react-query'; -import { upperFirst } from 'lodash-es'; import { Suspense, useEffect, useMemo, useState } from 'react'; import { useSearchParams } from 'react-router-dom'; +import { cn } from 'tailwind-preset'; import { Badge, Button, @@ -109,7 +109,7 @@ function Filters() { }} getDisplayValue={() => FILTER_SEARCHPARAMS['agentRunning']} > - {['On', 'Off'] + {['Yes', 'No'] .filter((item) => { if (!agentRunningSearchText.length) return true; return item.toLowerCase().includes(agentRunningSearchText.toLowerCase()); @@ -347,7 +347,7 @@ function useSearchClustersWithPagination() { clusterIds: searchParams.getAll('clusters'), agentRunning: searchParams .getAll('agentRunning') - .map((value) => (value === 'On' ? true : false)), + .map((value) => (value === 'Yes' ? true : false)), }), keepPreviousData: true, }); @@ -458,7 +458,14 @@ const DataTable = ({ }), columnHelper.accessor('agent_running', { cell: (info) => { - return ; + return ( + + ); }, header: () => Agent running, minSize: 60, diff --git a/deepfence_frontend/apps/dashboard/src/features/vulnerabilities/pages/MostExploitableVulnerabilities.tsx b/deepfence_frontend/apps/dashboard/src/features/vulnerabilities/pages/MostExploitableVulnerabilities.tsx index 8700651481..91a5723dff 100644 --- a/deepfence_frontend/apps/dashboard/src/features/vulnerabilities/pages/MostExploitableVulnerabilities.tsx +++ b/deepfence_frontend/apps/dashboard/src/features/vulnerabilities/pages/MostExploitableVulnerabilities.tsx @@ -50,7 +50,7 @@ import { ScanTypeEnum, VulnerabilitySeverityType } from '@/types/common'; import { get403Message } from '@/utils/403'; import { apiWrapper, retryUntilResponseHasValue } from '@/utils/api'; import { download } from '@/utils/download'; -import { SeverityEnumList } from '@/utils/enum'; +import { getSeverityPrettyName, SeverityEnumList, SeverityValueType } from '@/utils/enum'; const DEFAULT_PAGE_SIZE = 10; @@ -79,6 +79,24 @@ const FILTER_SEARCHPARAMS: Record = { clusters: 'Clusters', }; +const getPrettyNameForAppliedFilters = ({ + key, + value, +}: { + key: string; + value: string; +}) => { + switch (key) { + case 'severity': + return getSeverityPrettyName(value as SeverityValueType); + case 'liveConnection': + return upperFirst(value); + + default: + return value; + } +}; + enum ActionEnumType { DOWNLOAD = 'download', } @@ -399,7 +417,10 @@ const Filters = () => { ); diff --git a/deepfence_frontend/apps/dashboard/src/features/vulnerabilities/pages/UniqueVulnerabilities.tsx b/deepfence_frontend/apps/dashboard/src/features/vulnerabilities/pages/UniqueVulnerabilities.tsx index 93e16b81f9..e4be88d31a 100644 --- a/deepfence_frontend/apps/dashboard/src/features/vulnerabilities/pages/UniqueVulnerabilities.tsx +++ b/deepfence_frontend/apps/dashboard/src/features/vulnerabilities/pages/UniqueVulnerabilities.tsx @@ -35,7 +35,7 @@ import { FilterWrapper } from '@/features/common/FilterWrapper'; import { queries } from '@/queries'; import { useTheme } from '@/theme/ThemeContext'; import { ScanTypeEnum, VulnerabilitySeverityType } from '@/types/common'; -import { SeverityEnumList } from '@/utils/enum'; +import { getSeverityPrettyName, SeverityEnumList, SeverityValueType } from '@/utils/enum'; import { getOrderFromSearchParams, useSortingState } from '@/utils/table'; const DEFAULT_PAGE_SIZE = 10; @@ -65,6 +65,24 @@ const FILTER_SEARCHPARAMS: Record = { clusters: 'Clusters', }; +const getPrettyNameForAppliedFilters = ({ + key, + value, +}: { + key: string; + value: string; +}) => { + switch (key) { + case 'severity': + return getSeverityPrettyName(value as SeverityValueType); + case 'liveConnection': + return upperFirst(value); + + default: + return value; + } +}; + const getAppliedFiltersCount = (searchParams: URLSearchParams) => { return Object.keys(FILTER_SEARCHPARAMS).reduce((prev, curr) => { return prev + searchParams.getAll(curr).length; @@ -287,7 +305,10 @@ const Filters = () => { ); diff --git a/deepfence_frontend/apps/dashboard/src/features/vulnerabilities/pages/VulnerabilityScanResults.tsx b/deepfence_frontend/apps/dashboard/src/features/vulnerabilities/pages/VulnerabilityScanResults.tsx index 1234bfb5ea..e2ecd02e2a 100644 --- a/deepfence_frontend/apps/dashboard/src/features/vulnerabilities/pages/VulnerabilityScanResults.tsx +++ b/deepfence_frontend/apps/dashboard/src/features/vulnerabilities/pages/VulnerabilityScanResults.tsx @@ -98,7 +98,13 @@ import { import { get403Message, getResponseErrors } from '@/utils/403'; import { apiWrapper } from '@/utils/api'; import { formatMilliseconds } from '@/utils/date'; -import { SeverityEnum, SeverityEnumList } from '@/utils/enum'; +import { + getMaskedUnmaskedPrettyName, + getSeverityPrettyName, + SeverityEnum, + SeverityEnumList, + SeverityValueType, +} from '@/utils/enum'; import { abbreviateNumber } from '@/utils/number'; import { isScanComplete, @@ -1216,6 +1222,30 @@ const FILTER_SEARCHPARAMS: Record = { language: 'CVE Type', exploitable: 'Exploitable', }; + +const getPrettyNameForAppliedFilters = ({ + key, + value, +}: { + key: string; + value: string; +}) => { + switch (key) { + case 'severity': + return getSeverityPrettyName(value as SeverityValueType); + case 'visibility': + return getMaskedUnmaskedPrettyName(value); + case 'exploitable': + return 'Most Exploitable'; + case 'attackVector': + return capitalize(value); + case 'language': + return capitalize(value); + default: + return value; + } +}; + const getAppliedFiltersCount = (searchParams: URLSearchParams) => { return Object.keys(FILTER_SEARCHPARAMS).reduce((prev, curr) => { return prev + searchParams.getAll(curr).length; @@ -1306,7 +1336,7 @@ const Filters = () => { }).map((item) => { return ( - {capitalize(item)} + {getSeverityPrettyName(item)} ); })} @@ -1383,7 +1413,7 @@ const Filters = () => { return prev; }); }} - text={`${FILTER_SEARCHPARAMS[key]}: ${value}`} + text={`${FILTER_SEARCHPARAMS[key]}: ${getPrettyNameForAppliedFilters({ key, value })}`} /> ); })} diff --git a/deepfence_frontend/apps/dashboard/src/features/vulnerabilities/pages/VulnerabilityScans.tsx b/deepfence_frontend/apps/dashboard/src/features/vulnerabilities/pages/VulnerabilityScans.tsx index 958bbed8eb..b9fc62be2d 100644 --- a/deepfence_frontend/apps/dashboard/src/features/vulnerabilities/pages/VulnerabilityScans.tsx +++ b/deepfence_frontend/apps/dashboard/src/features/vulnerabilities/pages/VulnerabilityScans.tsx @@ -1,6 +1,6 @@ import { useSuspenseQuery } from '@suspensive/react-query'; import { useIsFetching } from '@tanstack/react-query'; -import { capitalize } from 'lodash-es'; +import { capitalize, startCase } from 'lodash-es'; import React, { Suspense, useCallback, useEffect, useMemo, useState } from 'react'; import { ActionFunctionArgs, @@ -788,6 +788,21 @@ const FILTER_SEARCHPARAMS: Record = { registryAccounts: 'Registry', }; +const getPrettyNameForAppliedFilters = ({ + key, + value, +}: { + key: string; + value: string; +}) => { + switch (key) { + case 'nodeType': + return startCase(value); + default: + return value; + } +}; + const getAppliedFiltersCount = (searchParams: URLSearchParams) => { return Object.keys(FILTER_SEARCHPARAMS).reduce((prev, curr) => { return prev + searchParams.getAll(curr).length; @@ -1036,7 +1051,10 @@ const Filters = () => { ); diff --git a/deepfence_frontend/apps/dashboard/src/utils/enum.ts b/deepfence_frontend/apps/dashboard/src/utils/enum.ts index b61d6e1a5a..c0830dedc0 100644 --- a/deepfence_frontend/apps/dashboard/src/utils/enum.ts +++ b/deepfence_frontend/apps/dashboard/src/utils/enum.ts @@ -97,3 +97,17 @@ export function getSeverityPrettyName(severity: SeverityValueType) { throw new Error(`Unhandled case: ${_exhaustiveCheck}`); } } + +export function getMaskedUnmaskedPrettyName(maskedUnmasked: string) { + switch (maskedUnmasked) { + case 'masked': + return 'Masked'; + case 'unmasked': + return 'Unmasked'; + + default: + throw new Error( + `Unhandled case: No matching key found for getMaskedUnmaskedPrettyName`, + ); + } +} diff --git a/deepfence_server/constants/api-messages/success.go b/deepfence_server/constants/api-messages/success.go index 96c38cb354..993f9db40d 100644 --- a/deepfence_server/constants/api-messages/success.go +++ b/deepfence_server/constants/api-messages/success.go @@ -14,4 +14,5 @@ const ( ErrIntegrationTypeCannotBeUpdated = "integration type cannot be updated" ErrIntegrationTypeEmpty = "integration type cannot be empty" ErrNotificationTypeEmpty = "notification type cannot be empty" + ErrSendSummaryNotSupported = "notification type does not support sending summary links" ) diff --git a/deepfence_server/controls/agent.go b/deepfence_server/controls/agent.go index e7e76706f9..deaa03d41d 100644 --- a/deepfence_server/controls/agent.go +++ b/deepfence_server/controls/agent.go @@ -69,7 +69,7 @@ func GetAgentActions(ctx context.Context, agentID model.AgentID, consoleURL stri actions = append(actions, upgradeActions...) } - scanActions, scanErr := ExtractStartingAgentScans(ctx, nodeID, workNumToExtract) + scanActions, scanErr := ExtractStartingAgentScans(ctx, nodeID, agentType, workNumToExtract) workNumToExtract -= len(scanActions) if scanErr == nil { actions = append(actions, scanActions...) @@ -312,7 +312,7 @@ func hasPendingAgentScans(ctx context.Context, client neo4j.DriverWithContext, n return len(records) != 0, err } -func ExtractStartingAgentScans(ctx context.Context, nodeID string, maxWork int) ([]controls.Action, error) { +func ExtractStartingAgentScans(ctx context.Context, nodeID string, agentType string, maxWork int) ([]controls.Action, error) { ctx, span := telemetry.NewSpan(ctx, "control", "extract-starting-agent-scans") defer span.End() @@ -340,14 +340,28 @@ func ExtractStartingAgentScans(ctx context.Context, nodeID string, maxWork int) } defer tx.Close(ctx) - r, err := tx.Run(ctx, `MATCH (s) -[:SCHEDULED]-> (n:Node{node_id:$id}) + var r neo4j.ResultWithContext + if agentType == controls.CLOUD_AGENT { + r, err = tx.Run(ctx, `MATCH (c:CloudNode) <- [:SCANNED] - (s) -[:SCHEDULED]-> (n:Node{node_id:$id}) WHERE s.status = '`+utils.ScanStatusStarting+`' + AND c.refresh_status = 'COMPLETE' + AND c.active = true AND s.retries < 3 WITH s ORDER BY s.is_priority DESC, s.updated_at ASC LIMIT $max_work SET s.status = '`+utils.ScanStatusInProgress+`', s.updated_at = TIMESTAMP() WITH s RETURN s.trigger_action`, - map[string]interface{}{"id": nodeID, "max_work": maxWork}) + map[string]interface{}{"id": nodeID, "max_work": maxWork}) + } else { + r, err = tx.Run(ctx, `MATCH (s) -[:SCHEDULED]-> (n:Node{node_id:$id}) + WHERE s.status = '`+utils.ScanStatusStarting+`' + AND s.retries < 3 + WITH s ORDER BY s.is_priority DESC, s.updated_at ASC LIMIT $max_work + SET s.status = '`+utils.ScanStatusInProgress+`', s.updated_at = TIMESTAMP() + WITH s + RETURN s.trigger_action`, + map[string]interface{}{"id": nodeID, "max_work": maxWork}) + } if err != nil { return res, err @@ -629,8 +643,7 @@ func ExtractPendingAgentUpgrade(ctx context.Context, nodeID string, maxWork int, } -func ExtractRefreshResourceAction(ctx context.Context, nodeID string, - maxWork int) ([]controls.Action, error) { +func ExtractRefreshResourceAction(ctx context.Context, nodeID string, maxWork int) ([]controls.Action, error) { ctx, span := telemetry.NewSpan(ctx, "control", "extract-pending-refresh-resources") defer span.End() @@ -659,15 +672,15 @@ func ExtractRefreshResourceAction(ctx context.Context, nodeID string, defer tx.Close(ctx) r, err := tx.Run(ctx, ` - MATCH(n:Node{node_id:$id}) -[:HOSTS]-> (c:CloudNode) - MATCH(r:CloudNodeRefresh{node_id:c.node_id}) - WHERE r.refresh=true - WITH HEAD(collect(r)) AS rnode - WHERE rnode IS NOT NULL - WITH rnode, rnode.node_id AS node_id - DETACH DELETE rnode - RETURN node_id`, - map[string]interface{}{"id": nodeID}) + MATCH(n:Node{node_id:$id}) -[:HOSTS]-> (r:CloudNode) + WHERE r.refresh_status = '`+utils.ScanStatusStarting+`' + AND r.cloud_provider in ['`+model.PostureProviderAWS+`','`+model.PostureProviderGCP+`','`+model.PostureProviderAzure+`'] + AND COALESCE(r.cloud_compliance_scan_status, '') <> '`+utils.ScanStatusInProgress+`' + WITH r LIMIT $max_work + SET r.refresh_status = '`+utils.ScanStatusInProgress+`', r.refresh_message = '' + WITH r + RETURN r.node_id, r.node_name AS account_id`, + map[string]interface{}{"id": nodeID, "max_work": maxWork}) if err != nil { return res, err @@ -683,13 +696,14 @@ func ExtractRefreshResourceAction(ctx context.Context, nodeID string, for _, record := range records { var action controls.Action - if record.Values[0] == nil { + if record.Values[0] == nil || record.Values[1] == nil { log.Error().Msgf("Invalid CloudNode ID, skipping") continue } req := controls.RefreshResourcesRequest{} req.NodeId = record.Values[0].(string) + req.AccountID = record.Values[1].(string) req.NodeType = controls.CloudAccount reqBytes, err := json.Marshal(req) diff --git a/deepfence_server/go.mod b/deepfence_server/go.mod index 4015b45fa2..9c8a42ec1b 100644 --- a/deepfence_server/go.mod +++ b/deepfence_server/go.mod @@ -134,7 +134,7 @@ require ( github.com/segmentio/asm v1.2.0 // indirect github.com/sendgrid/sendgrid-go v3.14.0+incompatible github.com/shopspring/decimal v1.2.0 // indirect - github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/cast v1.6.0 github.com/spf13/pflag v1.0.5 // indirect github.com/swaggest/jsonschema-go v0.3.64 // indirect github.com/swaggest/refl v1.3.0 // indirect diff --git a/deepfence_server/handler/cloud_node.go b/deepfence_server/handler/cloud_node.go index 6e4e5585fd..a86d39bebf 100644 --- a/deepfence_server/handler/cloud_node.go +++ b/deepfence_server/handler/cloud_node.go @@ -8,6 +8,7 @@ import ( "net/http" "github.com/deepfence/ThreatMapper/deepfence_server/model" + "github.com/deepfence/ThreatMapper/deepfence_server/reporters" reporters_scan "github.com/deepfence/ThreatMapper/deepfence_server/reporters/scan" ctl "github.com/deepfence/ThreatMapper/deepfence_utils/controls" "github.com/deepfence/ThreatMapper/deepfence_utils/directory" @@ -19,6 +20,9 @@ import ( var ( cloudAccountNodeType = ctl.ResourceTypeToString(ctl.CloudAccount) + refreshAccountFilter = reporters.FieldsFilters{ + NotContainsFilter: reporters.ContainsFilter{FieldsValues: map[string][]interface{}{"refresh_status": {utils.ScanStatusStarting}}}, + } ) func (h *Handler) RegisterCloudNodeAccountHandler(w http.ResponseWriter, r *http.Request) { @@ -59,19 +63,20 @@ func (h *Handler) RegisterCloudNodeAccountHandler(w http.ResponseWriter, r *http "version": req.Version, "node_type": req.CloudProvider, } - err = model.UpsertCloudComplianceNode(ctx, orgAccountNode, "", req.HostNodeID) + err = model.UpsertCloudAccount(ctx, orgAccountNode, req.IsOrganizationDeployment, req.HostNodeID) if err != nil { h.complianceError(w, err.Error()) return } - for _, monitoredAccount := range monitoredAccounts { + monitoredNodes := make([]map[string]interface{}, len(monitoredAccounts)) + for i, monitoredAccount := range monitoredAccounts { err = h.Validator.Struct(monitoredAccount) if err != nil { log.Error().Msg(err.Error()) h.respondError(&ValidatorError{err: err}, w) return } - monitoredNode := map[string]interface{}{ + monitoredNodes[i] = map[string]interface{}{ "node_id": monitoredAccount.NodeID, "cloud_provider": req.CloudProvider, "node_name": monitoredAccount.AccountID, @@ -80,12 +85,12 @@ func (h *Handler) RegisterCloudNodeAccountHandler(w http.ResponseWriter, r *http "version": req.Version, "node_type": req.CloudProvider, } - err = model.UpsertCloudComplianceNode(ctx, monitoredNode, orgNodeID, req.HostNodeID) - if err != nil { - log.Error().Msgf("Error while upserting node: %+v", err) - h.complianceError(w, err.Error()) - return - } + } + err = model.UpsertChildCloudAccounts(ctx, monitoredNodes, orgNodeID, req.HostNodeID) + if err != nil { + log.Error().Msgf("Error while upserting node: %+v", err) + h.complianceError(w, err.Error()) + return } } else { log.Debug().Msgf("Single account monitoring for node: %s", nodeID) @@ -98,7 +103,7 @@ func (h *Handler) RegisterCloudNodeAccountHandler(w http.ResponseWriter, r *http "node_type": req.CloudProvider, } log.Debug().Msgf("Node for upsert: %+v", node) - err = model.UpsertCloudComplianceNode(ctx, node, "", req.HostNodeID) + err = model.UpsertCloudAccount(ctx, node, req.IsOrganizationDeployment, req.HostNodeID) if err != nil { log.Error().Msgf("Error while upserting node: %+v", err) h.complianceError(w, err.Error()) @@ -124,13 +129,19 @@ func (h *Handler) RefreshCloudAccountHandler(w http.ResponseWriter, r *http.Requ nodeIdentifiers[i] = model.NodeIdentifier{NodeID: id, NodeType: cloudAccountNodeType} } - cloudNodeIds, err := reporters_scan.GetCloudAccountIDs(r.Context(), nodeIdentifiers) + cloudNodeIds, err := reporters_scan.GetCloudAccountIDs(r.Context(), nodeIdentifiers, &refreshAccountFilter) if err != nil { log.Error().Msgf(err.Error()) h.respondError(&BadDecoding{err}, w) return } + if len(cloudNodeIds) == 0 { + // Refresh already in progress for all requested cloud accounts + w.WriteHeader(http.StatusNoContent) + return + } + resolvedRequest := model.CloudAccountRefreshReq{NodeIDs: make([]string, len(cloudNodeIds))} for i, id := range cloudNodeIds { resolvedRequest.NodeIDs[i] = id.NodeID diff --git a/deepfence_server/handler/integration.go b/deepfence_server/handler/integration.go index 1368d83cea..d68ca1dd72 100644 --- a/deepfence_server/handler/integration.go +++ b/deepfence_server/handler/integration.go @@ -44,6 +44,18 @@ func (h *Handler) AddIntegration(w http.ResponseWriter, r *http.Request) { return } + if req.SendSummary { + if integration.SupportsSummaryLink(req.IntegrationType) { + req.Config["send_summary"] = True + } else { + err = httpext.JSON(w, http.StatusBadRequest, model.ErrorResponse{Message: api_messages.ErrSendSummaryNotSupported}) + if err != nil { + log.Error().Msg(err.Error()) + } + return + } + } + req.Config["filter_hash"], err = GetFilterHash(req.Filters) if err != nil { log.Error().Msgf("%v", err) @@ -286,6 +298,18 @@ func (h *Handler) UpdateIntegration(w http.ResponseWriter, r *http.Request) { return } + if req.SendSummary { + if integration.SupportsSummaryLink(req.IntegrationType) { + req.Config["send_summary"] = True + } else { + err = httpext.JSON(w, http.StatusBadRequest, model.ErrorResponse{Message: api_messages.ErrSendSummaryNotSupported}) + if err != nil { + log.Error().Msg(err.Error()) + } + return + } + } + // check if integration is valid /*err = i.SendNotification("validating integration") if err != nil { diff --git a/deepfence_server/handler/scan_reports.go b/deepfence_server/handler/scan_reports.go index d2b1a66a7f..e8e17683c4 100644 --- a/deepfence_server/handler/scan_reports.go +++ b/deepfence_server/handler/scan_reports.go @@ -418,7 +418,7 @@ func (h *Handler) StartComplianceScanHandler(w http.ResponseWriter, r *http.Requ regular, k8s, _, _ := extractBulksNodes(reqs.NodeIDs) - cloudNodeIds, err := reportersScan.GetCloudAccountIDs(ctx, regular) + cloudNodeIds, err := reportersScan.GetCloudAccountIDs(ctx, regular, nil) if err != nil { h.respondError(err, w) return diff --git a/deepfence_server/ingesters/scan_status.go b/deepfence_server/ingesters/scan_status.go index 068b8fb63b..d462bf6049 100644 --- a/deepfence_server/ingesters/scan_status.go +++ b/deepfence_server/ingesters/scan_status.go @@ -325,16 +325,16 @@ func AddNewCloudComplianceScan( } } } - nt := controls.KubernetesCluster - if nodeType == controls.ResourceTypeToString(controls.Host) { - nt = controls.Host - } var action []byte var hostNodeID, hostNeo4jNodeType string if nodeType == controls.ResourceTypeToString(controls.KubernetesCluster) || nodeType == controls.ResourceTypeToString(controls.Host) { hostNodeID = nodeID hostNeo4jNodeType = neo4jNodeType + nt := controls.KubernetesCluster + if nodeType == controls.ResourceTypeToString(controls.Host) { + nt = controls.Host + } internalReq, _ := json.Marshal(controls.StartComplianceScanRequest{ NodeID: nodeID, NodeType: nt, @@ -374,7 +374,7 @@ func AddNewCloudComplianceScan( internalReq, _ := json.Marshal(controls.StartCloudComplianceScanRequest{ NodeID: nodeID, - NodeType: nt, + NodeType: controls.CloudAccount, BinArgs: map[string]string{"scan_id": scanID, "benchmark_types": strings.Join(benchmarkTypes, ",")}, ScanDetails: controls.CloudComplianceScanDetails{ ScanId: scanID, diff --git a/deepfence_server/model/cloud_node.go b/deepfence_server/model/cloud_node.go index 8a74aff137..bf1f1c57dd 100644 --- a/deepfence_server/model/cloud_node.go +++ b/deepfence_server/model/cloud_node.go @@ -84,7 +84,8 @@ type CloudNodeAccountInfo struct { LastScanID string `json:"last_scan_id"` LastScanStatus string `json:"last_scan_status"` RefreshMessage string `json:"refresh_message"` - RefreshStatus string `json:"refresh_status" enum:"STARTING,IN_PROGRESS,ERROR,COMPLETE"` + RefreshStatus string `json:"refresh_status"` + RefreshStatusMap map[string]int64 `json:"refresh_status_map"` ScanStatusMap map[string]int64 `json:"scan_status_map"` Version string `json:"version"` HostNodeID string `json:"host_node_id"` @@ -205,8 +206,7 @@ type PostureProvider struct { ResourceCount int64 `json:"resource_count"` } -func UpsertCloudComplianceNode(ctx context.Context, nodeDetails map[string]interface{}, - parentNodeID string, hostNodeID string) error { +func UpsertCloudAccount(ctx context.Context, nodeDetails map[string]interface{}, isOrganizationDeployment bool, hostNodeID string) error { ctx, span := telemetry.NewSpan(ctx, "model", "upsert-cloud-compliance-node") defer span.End() @@ -225,39 +225,65 @@ func UpsertCloudComplianceNode(ctx context.Context, nodeDetails map[string]inter } defer tx.Close(ctx) - if parentNodeID == "" { - if _, err := tx.Run(ctx, ` - MERGE (r:Node{node_id:$host_node_id, node_type: "cloud_agent"}) - WITH $param as row, r - MERGE (n:CloudNode{node_id:row.node_id}) - ON CREATE SET n.refresh_status = 'STARTING', n.refresh_message = '' - MERGE (r) -[:HOSTS]-> (n) - SET n+= row, n.active = true, n.updated_at = TIMESTAMP(), n.version = row.version, - r.node_name=$host_node_id, r.active = true, r.agent_running=true, r.updated_at = TIMESTAMP()`, - map[string]interface{}{ - "param": nodeDetails, - "host_node_id": hostNodeID, - }); err != nil { - return err - } - } else { - if _, err := tx.Run(ctx, ` + var setRefreshStatusQuery string + // Organization account node does not have refresh status. It is rather an aggregation of all child account statuses. + if !isOrganizationDeployment { + setRefreshStatusQuery = `ON CREATE SET n.refresh_status = '` + utils.ScanStatusStarting + `', n.refresh_message = ''` + } + + if _, err = tx.Run(ctx, ` + MERGE (r:Node{node_id:$host_node_id, node_type: "cloud_agent"}) + WITH $param as row, r + MERGE (n:CloudNode{node_id:row.node_id}) + `+setRefreshStatusQuery+` + MERGE (r) -[:HOSTS]-> (n) + SET n+= row, n.active = true, n.updated_at = TIMESTAMP(), n.version = row.version, + r.node_name=$host_node_id, r.active = true, r.agent_running=true, r.updated_at = TIMESTAMP()`, + map[string]interface{}{ + "param": nodeDetails, + "host_node_id": hostNodeID, + }); err != nil { + return err + } + + return tx.Commit(ctx) +} + +func UpsertChildCloudAccounts(ctx context.Context, nodeDetails []map[string]interface{}, parentNodeID string, hostNodeID string) error { + ctx, span := telemetry.NewSpan(ctx, "model", "upsert-cloud-compliance-node") + defer span.End() + + driver, err := directory.Neo4jClient(ctx) + if err != nil { + return err + } + + session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite}) + defer session.Close(ctx) + + tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) + if err != nil { + return err + } + defer tx.Close(ctx) + + if _, err = tx.Run(ctx, ` MERGE (r:Node{node_id:$host_node_id, node_type: "cloud_agent"}) MERGE (m:CloudNode{node_id: $parent_node_id}) - WITH $param as row, r, m + WITH r, m + UNWIND $param as row MERGE (n:CloudNode{node_id:row.node_id}) - ON CREATE SET n.refresh_status = 'STARTING', n.refresh_message = '' + ON CREATE SET n.refresh_status = '`+utils.ScanStatusStarting+`', n.refresh_message = '' MERGE (m) -[:IS_CHILD]-> (n) MERGE (r) -[:HOSTS]-> (n) SET n+= row, n.active = true, n.updated_at = TIMESTAMP(), n.version = row.version, r.active = true, r.agent_running=true, r.updated_at = TIMESTAMP()`, - map[string]interface{}{ - "param": nodeDetails, - "parent_node_id": parentNodeID, - "host_node_id": hostNodeID, - }); err != nil { - return err - } + map[string]interface{}{ + "param": nodeDetails, + "parent_node_id": parentNodeID, + "host_node_id": hostNodeID, + }); err != nil { + return err } return tx.Commit(ctx) @@ -532,8 +558,8 @@ func (c *CloudAccountRefreshReq) SetCloudAccountRefresh(ctx context.Context) err if _, err = tx.Run(ctx, ` UNWIND $batch as cloudNode - MERGE (n:CloudNodeRefresh{node_id: cloudNode}) - SET n.refresh = true, n.updated_at = TIMESTAMP()`, + MATCH (m:CloudNode{node_id: cloudNode}) + SET m.refresh_status = '`+utils.ScanStatusStarting+`', m.refresh_message = ''`, map[string]interface{}{ "batch": c.NodeIDs, }); err != nil { @@ -542,50 +568,6 @@ func (c *CloudAccountRefreshReq) SetCloudAccountRefresh(ctx context.Context) err return tx.Commit(ctx) } -func (c *CloudAccountRefreshReq) GetCloudAccountRefresh(ctx context.Context) ([]string, error) { - - ctx, span := telemetry.NewSpan(ctx, "model", "get-cloud-account-refresh") - defer span.End() - - var updatedNodeIDs []string - driver, err := directory.Neo4jClient(ctx) - if err != nil { - return updatedNodeIDs, err - } - - session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite}) - defer session.Close(ctx) - - tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) - if err != nil { - return updatedNodeIDs, err - } - defer tx.Close(ctx) - - res, err := tx.Run(ctx, ` - UNWIND $batch as cloudNode - MATCH (n:CloudNodeRefresh{node_id: cloudNode}) - WHERE n.refresh=true - WITH n, n.node_id as deletedNodeID - DELETE n - RETURN deletedNodeID`, - map[string]interface{}{ - "batch": c.NodeIDs, - }) - if err != nil { - return updatedNodeIDs, err - } - recs, err := res.Collect(ctx) - if err != nil { - return updatedNodeIDs, err - } - - for _, rec := range recs { - updatedNodeIDs = append(updatedNodeIDs, rec.Values[0].(string)) - } - return updatedNodeIDs, tx.Commit(ctx) -} - type CloudAccountDeleteReq struct { NodeIDs []string `json:"node_ids" validate:"required,gt=0" required:"true"` } diff --git a/deepfence_server/model/integration.go b/deepfence_server/model/integration.go index 5a8491b3c7..5ceffaf1f4 100644 --- a/deepfence_server/model/integration.go +++ b/deepfence_server/model/integration.go @@ -27,6 +27,7 @@ type IntegrationAddReq struct { IntegrationType string `json:"integration_type" required:"true"` NotificationType string `json:"notification_type" required:"true"` Filters IntegrationFilters `json:"filters"` + SendSummary bool `json:"send_summary"` } type IntegrationFilters struct { @@ -157,6 +158,7 @@ type IntegrationUpdateReq struct { NotificationType string `json:"notification_type"` Filters IntegrationFilters `json:"filters"` IntegrationID string `path:"integration_id" validate:"required" required:"true"` + SendSummary bool `json:"send_summary"` } func (i *IntegrationUpdateReq) UpdateIntegration(ctx context.Context, pgClient *postgresqlDb.Queries, integration postgresqlDb.Integration) error { diff --git a/deepfence_server/pkg/integration/aws-security-hub/awssecurityhub.go b/deepfence_server/pkg/integration/aws-security-hub/awssecurityhub.go index 22faf43040..0180209f31 100644 --- a/deepfence_server/pkg/integration/aws-security-hub/awssecurityhub.go +++ b/deepfence_server/pkg/integration/aws-security-hub/awssecurityhub.go @@ -433,3 +433,7 @@ func (a AwsSecurityHub) mapPayloadToFindings(msg []map[string]interface{}, resou func (a AwsSecurityHub) IsValidCredential(ctx context.Context) (bool, error) { return true, nil } + +func (a AwsSecurityHub) SendSummaryLink() bool { + return false +} diff --git a/deepfence_server/pkg/integration/elasticsearch/elasticsearch.go b/deepfence_server/pkg/integration/elasticsearch/elasticsearch.go index f93920bcef..b0b5f549f5 100644 --- a/deepfence_server/pkg/integration/elasticsearch/elasticsearch.go +++ b/deepfence_server/pkg/integration/elasticsearch/elasticsearch.go @@ -100,3 +100,7 @@ func (e ElasticSearch) IsValidCredential(ctx context.Context) (bool, error) { return true, nil } + +func (e ElasticSearch) SendSummaryLink() bool { + return false +} diff --git a/deepfence_server/pkg/integration/email/email.go b/deepfence_server/pkg/integration/email/email.go index e7d29e46b6..32aeaaee7f 100644 --- a/deepfence_server/pkg/integration/email/email.go +++ b/deepfence_server/pkg/integration/email/email.go @@ -6,8 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "sort" - "strconv" "strings" "github.com/deepfence/ThreatMapper/deepfence_server/model" @@ -35,69 +33,49 @@ func New(ctx context.Context, b []byte) (*Email, error) { return &h, nil } -func (e Email) FormatMessage(message []map[string]interface{}) string { - var entiremsg strings.Builder - entiremsg.WriteString("*") - entiremsg.WriteString(e.Resource) - entiremsg.WriteString("*\n\n") - - // Prepare the sorted keys so that the output has the records in same order - var keys []string - if len(message) > 0 { - keys = make([]string, 0, len(message[0])) - for key := range message[0] { - keys = append(keys, key) - } +func (e Email) FormatMessage(message []map[string]interface{}, + extras map[string]interface{}) (string, map[string][]byte) { - sort.Strings(keys) - } + var msg strings.Builder + var attachments = map[string][]byte{} - for k, v := range message { - entiremsg.WriteString("#") - entiremsg.WriteString(strconv.Itoa(k + 1)) - entiremsg.WriteString("\n") - for _, key := range keys { - if val, ok := v[key]; ok { - fmtVal := "" - if val != nil { - fmtVal = fmt.Sprintf("%v", val) + for k, v := range extras { + if v != "" { + if k == "severity_counts" { + s := "" + for i, j := range v.(map[string]int32) { + s = fmt.Sprintf(" %s: %d\r\n", i, j) } - entiremsg.WriteString(key) - entiremsg.WriteString(": ") - entiremsg.WriteString(fmtVal) - entiremsg.WriteString("\n") - delete(v, key) + msg.WriteString(fmt.Sprintf("%s:\r\n%s", k, s)) + } else { + msg.WriteString(fmt.Sprintf("%s: %v\r\n", k, v)) } } + } - // This is to handle if we have unprocessed data in the map - // Possilbe if all the records are not uniform - for key, val := range v { - fmtVal := "" - if val != nil { - fmtVal = fmt.Sprintf("%v", val) - } - entiremsg.WriteString(key) - entiremsg.WriteString(": ") - entiremsg.WriteString(fmtVal) - entiremsg.WriteString("\n") - } - entiremsg.WriteString("\n") + r, err := json.Marshal(message) + if err != nil { + log.Error().Err(err).Msg("failed to marshal results") } - return entiremsg.String() + + attachments["scan-results.json"] = r + + return msg.String(), attachments } -func (e Email) SendNotification(ctx context.Context, message []map[string]interface{}, extras map[string]interface{}) error { +func (e Email) SendNotification(ctx context.Context, + message []map[string]interface{}, extras map[string]interface{}) error { _, span := telemetry.NewSpan(ctx, "integrations", "email-send-notification") defer span.End() - m := e.FormatMessage(message) + m, a := e.FormatMessage(message, extras) emailSender, err := sendemail.NewEmailSender(ctx) if err != nil { return err } - return emailSender.Send([]string{e.Config.EmailID}, "Deepfence Subscription", m, "", nil) + return emailSender.Send([]string{e.Config.EmailID}, + fmt.Sprintf("Deepfence %s Subscription", e.Resource), m, "", a) } func (e Email) IsEmailConfigured(ctx context.Context) bool { @@ -127,3 +105,7 @@ func (e Email) IsEmailConfigured(ctx context.Context) bool { func (e Email) IsValidCredential(ctx context.Context) (bool, error) { return true, nil } + +func (e Email) SendSummaryLink() bool { + return e.Config.SendSummary +} diff --git a/deepfence_server/pkg/integration/email/types.go b/deepfence_server/pkg/integration/email/types.go index cb3e08a951..dad186bcd2 100644 --- a/deepfence_server/pkg/integration/email/types.go +++ b/deepfence_server/pkg/integration/email/types.go @@ -15,7 +15,8 @@ type Email struct { } type Config struct { - EmailID string `json:"email_id" validate:"required,email" required:"true"` + EmailID string `json:"email_id" validate:"required,email" required:"true"` + SendSummary bool `json:"send_summary"` } func (e Email) ValidateConfig(validate *validator.Validate) error { diff --git a/deepfence_server/pkg/integration/google-chronicle/googlechronicle.go b/deepfence_server/pkg/integration/google-chronicle/googlechronicle.go index dba2421d5e..f7838fa60b 100644 --- a/deepfence_server/pkg/integration/google-chronicle/googlechronicle.go +++ b/deepfence_server/pkg/integration/google-chronicle/googlechronicle.go @@ -65,3 +65,7 @@ func (g GoogleChronicle) SendNotification(ctx context.Context, message []map[str func (g GoogleChronicle) IsValidCredential(ctx context.Context) (bool, error) { return true, nil } + +func (g GoogleChronicle) SendSummaryLink() bool { + return false +} diff --git a/deepfence_server/pkg/integration/http-endpoint/http-endpoint.go b/deepfence_server/pkg/integration/http-endpoint/http-endpoint.go index 672a62d28f..5e068dcfd8 100644 --- a/deepfence_server/pkg/integration/http-endpoint/http-endpoint.go +++ b/deepfence_server/pkg/integration/http-endpoint/http-endpoint.go @@ -103,3 +103,7 @@ func (h HTTPEndpoint) IsValidCredential(ctx context.Context) (bool, error) { // Check the response status code. return resp.StatusCode == http.StatusOK, nil } + +func (h HTTPEndpoint) SendSummaryLink() bool { + return false +} diff --git a/deepfence_server/pkg/integration/integration.go b/deepfence_server/pkg/integration/integration.go index e451d9e867..a26e6eb581 100644 --- a/deepfence_server/pkg/integration/integration.go +++ b/deepfence_server/pkg/integration/integration.go @@ -58,17 +58,31 @@ func GetIntegration(ctx context.Context, integrationType string, b []byte) (Inte func IsMessagingFormat(integrationType string) bool { retVal := false switch integrationType { - case constants.Slack, constants.Teams, constants.PagerDuty, - constants.Email, constants.Jira: + case constants.Slack, + constants.Teams, + constants.PagerDuty, + constants.Email, + constants.Jira: retVal = true } return retVal } +func SupportsSummaryLink(integrationType string) bool { + switch integrationType { + case constants.Slack, + constants.Teams: + return true + } + return false +} + // Integration is the interface for all integrations type Integration interface { // extras are additional fields that are not part of the message + // if SendSummaryLink is true then message array is expected to be scan summay SendNotification(ctx context.Context, message []map[string]interface{}, extras map[string]interface{}) error ValidateConfig(*validator.Validate) error IsValidCredential(ctx context.Context) (bool, error) + SendSummaryLink() bool } diff --git a/deepfence_server/pkg/integration/jira/jira.go b/deepfence_server/pkg/integration/jira/jira.go index ecf545bf82..1f4cb4c6f1 100644 --- a/deepfence_server/pkg/integration/jira/jira.go +++ b/deepfence_server/pkg/integration/jira/jira.go @@ -53,7 +53,15 @@ func (j Jira) SendNotification(ctx context.Context, message []map[string]interfa extraStr := []string{} for k, v := range extras { if v != "" { - extraStr = append(extraStr, fmt.Sprintf("%s: %v", k, v)) + if k == "severity_counts" { + s := "" + for i, j := range v.(map[string]int32) { + s = fmt.Sprintf(" %s: %d\n", i, j) + } + extraStr = append(extraStr, fmt.Sprintf("%s:\n%s", k, s)) + } else { + extraStr = append(extraStr, fmt.Sprintf("%s: %v\n", k, v)) + } } } @@ -168,3 +176,7 @@ func (j Jira) IsValidCredential(ctx context.Context) (bool, error) { return true, nil } + +func (j Jira) SendSummaryLink() bool { + return false +} diff --git a/deepfence_server/pkg/integration/pagerduty/pagerduty.go b/deepfence_server/pkg/integration/pagerduty/pagerduty.go index 7b6a65e8c3..3a4c81a5e1 100644 --- a/deepfence_server/pkg/integration/pagerduty/pagerduty.go +++ b/deepfence_server/pkg/integration/pagerduty/pagerduty.go @@ -158,3 +158,7 @@ func IsValidCreds(p PagerDuty) (bool, error) { func (p PagerDuty) IsValidCredential(ctx context.Context) (bool, error) { return true, nil } + +func (p PagerDuty) SendSummaryLink() bool { + return false +} diff --git a/deepfence_server/pkg/integration/s3/s3.go b/deepfence_server/pkg/integration/s3/s3.go index 7d4ec69f07..d0d8d02a2c 100644 --- a/deepfence_server/pkg/integration/s3/s3.go +++ b/deepfence_server/pkg/integration/s3/s3.go @@ -142,3 +142,7 @@ func (s S3) IsValidCredential(ctx context.Context) (bool, error) { return true, nil } + +func (s S3) SendSummaryLink() bool { + return false +} diff --git a/deepfence_server/pkg/integration/slack/slack.go b/deepfence_server/pkg/integration/slack/slack.go index 879ccc49ea..c74f26e932 100644 --- a/deepfence_server/pkg/integration/slack/slack.go +++ b/deepfence_server/pkg/integration/slack/slack.go @@ -6,11 +6,13 @@ import ( "encoding/json" "fmt" "net/http" + "time" intgerr "github.com/deepfence/ThreatMapper/deepfence_server/pkg/integration/errors" "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/telemetry" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" + "github.com/spf13/cast" ) const BatchSize = 5 @@ -120,6 +122,100 @@ func (s Slack) FormatMessage(message []map[string]interface{}, index int) []map[ return blocks } +func (s Slack) FormatSummaryMessage(message []map[string]interface{}) []map[string]interface{} { + + blocks := []map[string]interface{}{} + + for _, m := range message { + + // header + blocks = append( + blocks, + map[string]interface{}{ + "type": "header", + "text": map[string]interface{}{ + "type": "plain_text", + "text": fmt.Sprintf("Deepfence %s Scan", s.Resource), + "emoji": true, + }, + }, + ) + + // info section + startedAt := time.UnixMilli(cast.ToInt64(m["created_at"])).Format(time.RFC1123) + completedAt := time.UnixMilli(cast.ToInt64(m["updated_at"])).Format(time.RFC1123) + blocks = append( + blocks, + map[string]interface{}{ + "type": "section", + "fields": []map[string]interface{}{ + { + "type": "mrkdwn", + "text": fmt.Sprintf("*Node Type:*\n%v", m["node_type"]), + }, + { + "type": "mrkdwn", + "text": fmt.Sprintf("*Node Name:*\n%v", m["node_name"]), + }, + { + "type": "mrkdwn", + "text": fmt.Sprintf("*Started On:*\n %s", startedAt), + }, + { + "type": "mrkdwn", + "text": fmt.Sprintf("*Completed On:*\n %s", completedAt), + }, + }, + }, + ) + + // Severity section + var severity = "" + for k, v := range m["severity_counts"].(map[string]int32) { + severity += fmt.Sprintf(">_%s:_ %d\n", k, v) + } + + blocks = append( + blocks, + map[string]interface{}{ + "type": "section", + "fields": []map[string]interface{}{ + { + "type": "mrkdwn", + "text": fmt.Sprintf("*%s Severity Count:*\n%s", s.Resource, severity), + }, + }, + }, + ) + + // link + blocks = append( + blocks, + map[string]interface{}{ + "type": "section", + "fields": []map[string]interface{}{ + { + "type": "mrkdwn", + "text": fmt.Sprintf("<%v|*Click here*> to view scan results on Console", + m["scan_result_link"]), + }, + }, + }, + ) + + // add a divider + blocks = append( + blocks, + map[string]interface{}{ + "type": "divider", + }, + ) + + } + + return blocks +} + func (s Slack) SendNotification(ctx context.Context, message []map[string]interface{}, extras map[string]interface{}) error { _, span := telemetry.NewSpan(ctx, "integrations", "slack-send-notification") @@ -137,7 +233,14 @@ func (s Slack) SendNotification(ctx context.Context, message []map[string]interf batchMsg := message[startIdx:endIdx] - m := s.FormatMessage(batchMsg, startIdx+1) + var m []map[string]interface{} + + if s.SendSummaryLink() { + m = s.FormatSummaryMessage(batchMsg) + } else { + m = s.FormatMessage(batchMsg, startIdx+1) + } + payload := map[string]interface{}{ "blocks": m, } @@ -181,7 +284,8 @@ func (s Slack) SendNotification(ctx context.Context, message []map[string]interf } } resp.Body.Close() - return fmt.Errorf("failed to send notification batch %d, status code: %d , error: %s", i+1, resp.StatusCode, errorMsg) + return fmt.Errorf("failed to send notification batch %d, status code: %d , error: %s", + i+1, resp.StatusCode, errorMsg) } resp.Body.Close() } @@ -237,3 +341,7 @@ func (s Slack) IsValidCredential(ctx context.Context) (bool, error) { return true, nil } + +func (s Slack) SendSummaryLink() bool { + return s.Config.SendSummary +} diff --git a/deepfence_server/pkg/integration/slack/types.go b/deepfence_server/pkg/integration/slack/types.go index 2a003f24a6..de3fb1f2dc 100644 --- a/deepfence_server/pkg/integration/slack/types.go +++ b/deepfence_server/pkg/integration/slack/types.go @@ -19,8 +19,9 @@ func (s Slack) ValidateConfig(validate *validator.Validate) error { } type Config struct { - WebhookURL string `json:"webhook_url" validate:"required,url" required:"true"` - Channel string `json:"channel" validate:"required,min=1" required:"true"` + WebhookURL string `json:"webhook_url" validate:"required,url" required:"true"` + Channel string `json:"channel" validate:"required,min=1" required:"true"` + SendSummary bool `json:"send_summary"` } type Payload struct { diff --git a/deepfence_server/pkg/integration/splunk/splunk.go b/deepfence_server/pkg/integration/splunk/splunk.go index af8a8de083..3fbfe00987 100644 --- a/deepfence_server/pkg/integration/splunk/splunk.go +++ b/deepfence_server/pkg/integration/splunk/splunk.go @@ -120,3 +120,7 @@ func (s Splunk) Sender(in chan []byte, wg *sync.WaitGroup) { func (s Splunk) IsValidCredential(ctx context.Context) (bool, error) { return true, nil } + +func (s Splunk) SendSummaryLink() bool { + return false +} diff --git a/deepfence_server/pkg/integration/sumologic/sumologic.go b/deepfence_server/pkg/integration/sumologic/sumologic.go index 918aef605a..7cf53aeafb 100644 --- a/deepfence_server/pkg/integration/sumologic/sumologic.go +++ b/deepfence_server/pkg/integration/sumologic/sumologic.go @@ -76,3 +76,7 @@ func (s SumoLogic) SendNotification(ctx context.Context, data []map[string]inter func (s SumoLogic) IsValidCredential(ctx context.Context) (bool, error) { return true, nil } + +func (s SumoLogic) SendSummaryLink() bool { + return false +} diff --git a/deepfence_server/pkg/integration/teams/teams.go b/deepfence_server/pkg/integration/teams/teams.go index 09a8643356..c38072279d 100644 --- a/deepfence_server/pkg/integration/teams/teams.go +++ b/deepfence_server/pkg/integration/teams/teams.go @@ -9,10 +9,12 @@ import ( "strconv" "strings" "sync" + "time" "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/telemetry" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" + "github.com/spf13/cast" ) const BatchSize = 5 @@ -31,20 +33,21 @@ func (t Teams) FormatMessage(message map[string]interface{}, position int, entir if position == 1 { entiremsg.WriteString("**") entiremsg.WriteString(t.Resource) - entiremsg.WriteString("**

") + entiremsg.WriteString("**\n\n") } entiremsg.WriteString("**#") entiremsg.WriteString(strconv.Itoa(position)) - entiremsg.WriteString("**
") + entiremsg.WriteString("**\n") for key, val := range message { - entiremsg.WriteString(fmt.Sprintf("**%s**:%s
", key, val)) + entiremsg.WriteString(fmt.Sprintf("**%s**:%s\n", key, val)) } - entiremsg.WriteString("
") + entiremsg.WriteString("\n") return entiremsg.String() } -func (t Teams) SendNotification(ctx context.Context, message []map[string]interface{}, extras map[string]interface{}) error { +func (t Teams) SendNotification(ctx context.Context, + message []map[string]interface{}, extras map[string]interface{}) error { _, span := telemetry.NewSpan(ctx, "integrations", "teams-send-notification") defer span.End() @@ -58,7 +61,7 @@ func (t Teams) SendNotification(ctx context.Context, message []map[string]interf numRoutines = 10 } - senderChan := make(chan *Payload, 500) + senderChan := make(chan any, 500) var wg sync.WaitGroup for i := 0; i < numRoutines; i++ { wg.Add(1) @@ -80,13 +83,16 @@ func (t Teams) SendNotification(ctx context.Context, message []map[string]interf if endIndex > len(message) { endIndex = len(message) } - t.enqueueNotification(message[startIndex:endIndex], senderChan) + if !t.SendSummaryLink() { + t.enqueueNotification(message[startIndex:endIndex], senderChan) + } else { + t.enqueueSummaryNotification(message[startIndex:endIndex], senderChan) + } } return nil } -func (t Teams) enqueueNotification(payloads []map[string]interface{}, - senderChan chan *Payload) { +func (t Teams) enqueueNotification(payloads []map[string]interface{}, senderChan chan any) { var message strings.Builder var b strings.Builder @@ -103,7 +109,85 @@ func (t Teams) enqueueNotification(payloads []map[string]interface{}, senderChan <- &payload } -func (t Teams) Sender(in chan *Payload, wg *sync.WaitGroup) { +func (t Teams) enqueueSummaryNotification(payloads []map[string]interface{}, senderChan chan any) { + + for _, m := range payloads { + + ac := NewAdaptiveCards() + + body := []map[string]interface{}{} + + // header + body = append( + body, + map[string]interface{}{ + "type": "TextBlock", + "size": "medium", + "weight": "bolder", + "style": "heading", + "text": fmt.Sprintf("Deepfence %s Scan", t.Resource), + }, + ) + + // info + startedAt := time.UnixMilli(cast.ToInt64(m["created_at"])).Format(time.RFC1123) + completedAt := time.UnixMilli(cast.ToInt64(m["updated_at"])).Format(time.RFC1123) + body = append( + body, + map[string]interface{}{ + "type": "TextBlock", + "text": fmt.Sprintf("**Node Type:** %s", m["node_type"]), + "warp": "true", + }, + map[string]interface{}{ + "type": "TextBlock", + "text": fmt.Sprintf("**Node Name:** %s", m["node_name"]), + "warp": "true", + }, + map[string]interface{}{ + "type": "TextBlock", + "text": fmt.Sprintf("**Started On:** %s", startedAt), + "warp": "true", + }, + map[string]interface{}{ + "type": "TextBlock", + "text": fmt.Sprintf("**Completed On:** %s", completedAt), + "warp": "true", + }, + ) + + // Severity section + var severity = "" + for k, v := range m["severity_counts"].(map[string]int32) { + severity += fmt.Sprintf("> _%s:_ %d\n", k, v) + } + + body = append( + body, + map[string]interface{}{ + "type": "TextBlock", + "text": fmt.Sprintf("**%s Severity Count:**\n%s", t.Resource, severity), + "warp": "true", + }, + ) + + body = append( + body, + map[string]interface{}{ + "type": "TextBlock", + "text": fmt.Sprintf("[Click here](%s) to view scan results on Console", m["scan_result_link"]), + }, + ) + + ac.AddAttachment(NewAttachment(NewContent(body, nil))) + + // send summary card + senderChan <- &ac + } + +} + +func (t Teams) Sender(in chan any, wg *sync.WaitGroup) { defer wg.Done() for { @@ -173,3 +257,7 @@ func (t Teams) IsValidCredential(ctx context.Context) (bool, error) { return true, nil } + +func (t Teams) SendSummaryLink() bool { + return t.Config.SendSummary +} diff --git a/deepfence_server/pkg/integration/teams/types.go b/deepfence_server/pkg/integration/teams/types.go index 6506b585dc..bb60139162 100644 --- a/deepfence_server/pkg/integration/teams/types.go +++ b/deepfence_server/pkg/integration/teams/types.go @@ -18,7 +18,8 @@ type Teams struct { } type Config struct { - WebhookURL string `json:"webhook_url" validate:"required,url" required:"true"` + WebhookURL string `json:"webhook_url" validate:"required,url" required:"true"` + SendSummary bool `json:"send_summary"` } // Payloads @@ -48,3 +49,51 @@ type fact struct { Key string `json:"key"` Value string `json:"value"` } + +type AdaptiveCards struct { + Type string `json:"type"` + Attachments []Attachment `json:"attachments"` +} + +type Attachment struct { + ContentType string `json:"contentType"` + ContentURL any `json:"contentUrl"` + Content Content `json:"content"` +} + +type Content struct { + Schema string `json:"$schema"` + Type string `json:"type"` + Version string `json:"version"` + Body []map[string]interface{} `json:"body"` + Actions []map[string]interface{} `json:"action"` +} + +func NewAdaptiveCards() *AdaptiveCards { + return &AdaptiveCards{ + Type: "message", + Attachments: []Attachment{}, + } +} + +func (ac *AdaptiveCards) AddAttachment(attach Attachment) *AdaptiveCards { + ac.Attachments = append(ac.Attachments, attach) + return ac +} + +func NewAttachment(content Content) Attachment { + return Attachment{ + ContentType: "application/vnd.microsoft.card.adaptive", + ContentURL: nil, + Content: content, + } +} +func NewContent(body, actions []map[string]interface{}) Content { + return Content{ + Schema: "http://adaptivecards.io/schemas/adaptive-card.json", + Type: "AdaptiveCard", + Version: "1.0", + Body: body, + Actions: actions, + } +} diff --git a/deepfence_server/reporters/scan/scan_reporters.go b/deepfence_server/reporters/scan/scan_reporters.go index c2a759b303..d923d28174 100644 --- a/deepfence_server/reporters/scan/scan_reporters.go +++ b/deepfence_server/reporters/scan/scan_reporters.go @@ -398,7 +398,7 @@ func GetPodContainerIDs(ctx context.Context, podIds []model.NodeIdentifier) ([]m return res, nil } -func GetCloudAccountIDs(ctx context.Context, cloudProviderIds []model.NodeIdentifier) ([]model.NodeIdentifier, error) { +func GetCloudAccountIDs(ctx context.Context, cloudProviderIds []model.NodeIdentifier, filters *reporters.FieldsFilters) ([]model.NodeIdentifier, error) { ctx, span := telemetry.NewSpan(ctx, "scan-reports", "get-cloud-account-ids") defer span.End() @@ -418,9 +418,15 @@ func GetCloudAccountIDs(ctx context.Context, cloudProviderIds []model.NodeIdenti } defer tx.Close(ctx) + filterClauses := mo.None[reporters.FieldsFilters]() + if filters != nil { + filterClauses = mo.Some(*filters) + } + nres, err := tx.Run(ctx, ` MATCH (n:CloudNode) WHERE n.node_id IN $node_ids + `+reporters.ParseFieldFilters2CypherWhereConditions(`n`, filterClauses, false)+` RETURN n.node_id, n.cloud_provider`, map[string]interface{}{"node_ids": NodeIdentifierToIDList(cloudProviderIds)}) if err != nil { @@ -432,6 +438,7 @@ func GetCloudAccountIDs(ctx context.Context, cloudProviderIds []model.NodeIdenti return res, err } orgNodeIds := []string{} + childNodeIDs := []string{} for _, rec := range recs { cloudProvider := rec.Values[1].(string) if cloudProvider == model.PostureProviderAWSOrg || cloudProvider == model.PostureProviderGCPOrg || cloudProvider == model.PostureProviderAzureOrg { @@ -442,13 +449,16 @@ func GetCloudAccountIDs(ctx context.Context, cloudProviderIds []model.NodeIdenti NodeID: rec.Values[0].(string), NodeType: controls.ResourceTypeToString(controls.CloudAccount), }) + childNodeIDs = append(childNodeIDs, rec.Values[0].(string)) } if len(orgNodeIds) > 0 { nres, err = tx.Run(ctx, ` MATCH (n:CloudNode) -[:IS_CHILD] -> (m) - WHERE n.node_id IN $node_ids + WHERE n.node_id IN $node_ids + AND NOT m.node_id IN $child_node_ids + `+reporters.ParseFieldFilters2CypherWhereConditions(`m`, filterClauses, false)+` RETURN m.node_id`, - map[string]interface{}{"node_ids": orgNodeIds}) + map[string]interface{}{"node_ids": orgNodeIds, "child_node_ids": childNodeIDs}) if err != nil { return res, err } diff --git a/deepfence_server/reporters/search/search.go b/deepfence_server/reporters/search/search.go index e51e3d0b03..db68ef0e56 100644 --- a/deepfence_server/reporters/search/search.go +++ b/deepfence_server/reporters/search/search.go @@ -444,6 +444,11 @@ func searchCloudNode(ctx context.Context, filter SearchFilter, fw model.FetchWin if err != nil { log.Error().Msgf("Error in populating status of org %v", err) } + + node.RefreshStatusMap, err = getRefreshStatusMap(ctx, node.NodeID) + if err != nil { + log.Error().Msgf("Error in populating refresh status of org %v", err) + } } res = append(res, node) } @@ -493,6 +498,42 @@ func getScanStatusMap(ctx context.Context, id string, cloudProvider string) (map return res, nil } +func getRefreshStatusMap(ctx context.Context, nodeID string) (map[string]int64, error) { + + ctx, span := telemetry.NewSpan(ctx, "search", "get-scan-status-map") + defer span.End() + + res := map[string]int64{} + driver, err := directory.Neo4jClient(ctx) + if err != nil { + return res, err + } + + session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead}) + defer session.Close(ctx) + + tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) + if err != nil { + return res, err + } + defer tx.Close(ctx) + query := `MATCH (n:CloudNode{node_id:"` + nodeID + `"}) -[:IS_CHILD] -> (m:CloudNode) + RETURN m.refresh_status, count(m.refresh_status) ` + r, err := tx.Run(ctx, query, map[string]interface{}{}) + if err != nil { + return res, err + } + + recs, err := r.Collect(ctx) + if err != nil { + return res, err + } + for _, rec := range recs { + res[rec.Values[0].(string)] = rec.Values[1].(int64) + } + return res, nil +} + type FakeCypher struct { nodeType string extended string diff --git a/deepfence_utils/controls/agent.go b/deepfence_utils/controls/agent.go index 49858bb795..7d74eabe7a 100644 --- a/deepfence_utils/controls/agent.go +++ b/deepfence_utils/controls/agent.go @@ -27,6 +27,7 @@ const ( StartCloudComplianceScan StopCloudComplianceScan CloudScannerJobCount + CloudScannerResourceRefreshCount ) type ScanResource int @@ -160,8 +161,9 @@ type CloudComplianceScanBenchmark struct { } type RefreshResourcesRequest struct { - NodeId string `json:"node_id" required:"true"` - NodeType ScanResource `json:"node_type" required:"true"` + NodeId string `json:"node_id" required:"true"` + AccountID string `json:"account_id" required:"true"` + NodeType ScanResource `json:"node_type" required:"true"` } type StopSecretScanRequest StartSecretScanRequest diff --git a/deepfence_utils/integrations/metrics.go b/deepfence_utils/integrations/metrics.go index 3120cfe220..f33f87f9c5 100644 --- a/deepfence_utils/integrations/metrics.go +++ b/deepfence_utils/integrations/metrics.go @@ -6,8 +6,9 @@ import ( ) type Metrics struct { - Ok int64 `json:"ok,omitempty"` - Error int64 `json:"error,omitempty"` + Ok int64 `json:"ok,omitempty"` + Error int64 `json:"error,omitempty"` + IsSummary bool `json:"is_summary,omitempty"` } func (m *Metrics) Update(n Metrics) *Metrics { @@ -17,6 +18,7 @@ func (m *Metrics) Update(n Metrics) *Metrics { if n.Error > 0 { m.Error += n.Error } + m.IsSummary = n.IsSummary return m } diff --git a/deepfence_utils/utils/ingesters/cloud_resource.go b/deepfence_utils/utils/ingesters/cloud_resource.go index a76792c436..f083fff2b5 100644 --- a/deepfence_utils/utils/ingesters/cloud_resource.go +++ b/deepfence_utils/utils/ingesters/cloud_resource.go @@ -192,6 +192,7 @@ type CloudResourceRefreshStatus struct { CloudNodeID string `json:"cloud_node_id"` RefreshMessage string `json:"refresh_message"` RefreshStatus string `json:"refresh_status"` + UpdatedAt int64 `json:"updated_at"` } func (c *CloudResourceRefreshStatus) ToMap() map[string]interface{} { @@ -199,5 +200,6 @@ func (c *CloudResourceRefreshStatus) ToMap() map[string]interface{} { "cloud_node_id": c.CloudNodeID, "refresh_message": c.RefreshMessage, "refresh_status": c.RefreshStatus, + "updated_at": c.UpdatedAt, } } diff --git a/deepfence_worker/cronjobs/neo4j.go b/deepfence_worker/cronjobs/neo4j.go index 1479f7470c..ce8479950a 100644 --- a/deepfence_worker/cronjobs/neo4j.go +++ b/deepfence_worker/cronjobs/neo4j.go @@ -23,6 +23,7 @@ const ( dbUpgradeTimeout = time.Minute * 30 defaultDBScannedResourceCleanUpTimeout = time.Hour * 24 * 30 dbCloudResourceCleanupTimeout = time.Hour * 24 + dbCloudResourceRefreshTimeout = time.Hour * 12 ecsTaskRunningStatus = "RUNNING" ecsTaskPublicEnabledConfig = "ENABLED" dbDeletionTimeThreshold = time.Hour @@ -401,6 +402,20 @@ func CleanUpDB(ctx context.Context, task *asynq.Task) error { return err } + if _, err = session.Run(ctx, ` + MATCH (n:CloudNode) + WHERE n.refresh_updated_at < TIMESTAMP()-$time_ms + AND n.active = true + AND NOT n.refresh_status IN ['`+utils.ScanStatusStarting+`','`+utils.ScanStatusInProgress+`'] + WITH n LIMIT 10000 + SET n.refresh_status = '`+utils.ScanStatusStarting+`', n.refresh_updated_at = TIMESTAMP()`, + map[string]interface{}{ + "time_ms": dbCloudResourceRefreshTimeout.Milliseconds(), + }, txConfig); err != nil { + log.Error().Msgf("Error in Clean up DB task: %v", err) + return err + } + if _, err = session.Run(ctx, ` MATCH (n:CloudResource) <-[:HOSTS]- (cr:CloudRegion) WHERE n.updated_at < TIMESTAMP()-$time_ms diff --git a/deepfence_worker/cronjobs/notification.go b/deepfence_worker/cronjobs/notification.go index 0204323d28..a3c2377de9 100644 --- a/deepfence_worker/cronjobs/notification.go +++ b/deepfence_worker/cronjobs/notification.go @@ -8,6 +8,7 @@ import ( "fmt" "os" "strconv" + "strings" "time" reporters_search "github.com/deepfence/ThreatMapper/deepfence_server/reporters/search" @@ -20,15 +21,12 @@ import ( "github.com/deepfence/ThreatMapper/deepfence_utils/integrations" "github.com/deepfence/ThreatMapper/deepfence_utils/log" postgresql_db "github.com/deepfence/ThreatMapper/deepfence_utils/postgresql/postgresql-db" + "github.com/deepfence/ThreatMapper/deepfence_utils/setting" "github.com/deepfence/ThreatMapper/deepfence_utils/telemetry" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" "github.com/hibiken/asynq" ) -const NOTIFICATION_INTERVAL = 60000 //in milliseconds - -var ErrUnsupportedScan = errors.New("unsupported scan type in integration") - var fieldsMap = map[string]map[string]string{ utils.ScanTypeDetectedNode[utils.NEO4JVulnerabilityScan]: { "cve_severity": "Severity", @@ -95,12 +93,13 @@ var fieldsMap = map[string]map[string]string{ }, } -const DefaultNotificationErrorBackoff = 15 * time.Minute +var NotificationErrorBackoff time.Duration -var ( - NotificationErrorBackoff time.Duration - // notificationLock sync.Mutex -) +var ErrUnsupportedScan = errors.New("unsupported scan type in integration") + +const DefaultNotificationInterval = 60000 // in milliseconds + +const DefaultNotificationErrorBackoff = 15 * time.Minute func init() { backoffTimeStr := os.Getenv("DEEPFENCE_NOTIFICATION_ERROR_BACKOFF_MINUTES") @@ -150,7 +149,7 @@ func TriggerSendNotifications(ctx context.Context, task *asynq.Task) error { } // check if any integrations are configured - if len(integrations) <= 0 { + if len(integrations) == 0 { log.Warn().Msg("No integrations configured to notify") return nil } @@ -215,8 +214,8 @@ func SendNotifications(ctx context.Context, task *asynq.Task) error { err = processForResource(ctx, intg) - update_row := err != nil || (intg.ErrorMsg.Valid && err == nil) - if update_row { + updateStatus := err != nil || (intg.ErrorMsg.Valid && err == nil) + if updateStatus { var params postgresql_db.UpdateIntegrationStatusParams if err != nil { log.Error().Msgf("Error on integration %v", err) @@ -268,7 +267,7 @@ func getLastEventUpdatedAt(eTime sql.NullTime, defaultTime time.Time) string { if eTime.Valid { return strconv.FormatInt(eTime.Time.UnixMilli(), 10) } - return strconv.FormatInt(defaultTime.UnixMilli()-NOTIFICATION_INTERVAL, 10) + return strconv.FormatInt(defaultTime.UnixMilli()-DefaultNotificationInterval, 10) } func processIntegration[T any](ctx context.Context, intg postgresql_db.Integration) error { @@ -298,13 +297,15 @@ func processIntegration[T any](ctx context.Context, intg postgresql_db.Integrati start := time.Now() + lastUpdatedAt := getLastEventUpdatedAt(intg.LastEventUpdatedAt, start) + filters.FieldsFilters = reporters.FieldsFilters{} filters.FieldsFilters.CompareFilters = append( filters.FieldsFilters.CompareFilters, reporters.CompareFilter{ FieldName: "updated_at", GreaterThan: true, - FieldValue: getLastEventUpdatedAt(intg.LastEventUpdatedAt, start), + FieldValue: lastUpdatedAt, }, ) filters.FieldsFilters.ContainsFilter = reporters.ContainsFilter{ @@ -329,7 +330,7 @@ func processIntegration[T any](ctx context.Context, intg postgresql_db.Integrati scansList = append(scansList, scansInfo...) } } - neo4j_query_nc := time.Since(profileStart) + neo4jQueryNc := time.Since(profileStart) profileStart = time.Now() if reqC != nil { @@ -341,9 +342,9 @@ func processIntegration[T any](ctx context.Context, intg postgresql_db.Integrati scansList = append(scansList, containerScanInfo...) } } - neo4j_query_c := time.Since(profileStart) + neo4jQueryC := time.Since(profileStart) - neo4j_query_default := time.Duration(0) + neo4jQueryDefault := time.Duration(0) if reqNC == nil && reqC == nil { profileStart = time.Now() reqDefault := reporters_search.SearchScanReq{} @@ -358,13 +359,12 @@ func processIntegration[T any](ctx context.Context, intg postgresql_db.Integrati } else if len(defaultScanInfo) > 0 { scansList = append(scansList, defaultScanInfo...) } - neo4j_query_default = time.Since(profileStart) + neo4jQueryDefault = time.Since(profileStart) } // nothing to notify if len(scansList) == 0 { - log.Info().Msgf("No %s scans after timestamp %d", - intg.Resource, intg.LastEventUpdatedAt.Time.UnixMilli()) + log.Info().Msgf("No %s scans after timestamp %s", intg.Resource, lastUpdatedAt) return nil } @@ -385,9 +385,23 @@ func processIntegration[T any](ctx context.Context, intg postgresql_db.Integrati // this var tracks the latest scan in the retrived scans var latestScanUpdatedAt int64 + // need sql client update status, event updated_at and metrics + pgClient, err := directory.PostgresClient(ctx) + if err != nil { + log.Error().Err(err).Msg("unable to get sql client") + return err + } + + consoleURL, err := setting.GetManagementConsoleURL(ctx, pgClient) + if err != nil { + log.Error().Err(err).Msg("failed to get console url") + return err + } + + // apply filters while fetching results for _, scan := range scansList { - //This map is required as we might have duplicates in the scansList - //for the containers + // This map is required as we might have + // duplicates in the scansList for the containers if _, ok := scanIDMap[scan.ScanID]; ok { continue } @@ -406,71 +420,137 @@ func processIntegration[T any](ctx context.Context, intg postgresql_db.Integrati return err } - totalQueryTime = totalQueryTime + time.Since(profileStart) + totalQueryTime += time.Since(profileStart) + // skip if no results if len(results) == 0 { - log.Info().Msgf("No Results filtered for scan id: %s with filters %+v", scan.ScanID, filters) + log.Info().Msgf("No Results filtered for scan id: %s with filters %+v", + scan.ScanID, filters) continue } - var updatedResults []map[string]interface{} - // inject node details to results - if integration.IsMessagingFormat(intg.IntegrationType) { - updatedResults = FormatForMessagingApps(results, intg.Resource) - } else { - updatedResults = []map[string]interface{}{} - for _, r := range results { - updatedResults = append(updatedResults, utils.ToMap[T](r)) + // check if scan summary has to be sent + if integrationModel.SendSummaryLink() { + + summary := utils.ToMap[any](scan) + summary["scan_result_link"] = scanResultLink(consoleURL, scan.NodeType, intg.Resource, scan.ScanID) + + // send summary + err = integrationModel.SendNotification( + ctx, + []map[string]interface{}{summary}, + map[string]interface{}{}, + ) + if err != nil { + updateIntegrationMetrics(ctx, pgClient, intg.ID, + integrations.Metrics{Ok: 0, Error: 1, IsSummary: true}) + return err + } else { + updateIntegrationMetrics(ctx, pgClient, intg.ID, + integrations.Metrics{Ok: 1, Error: 0, IsSummary: true}) } - } - updatedResults = injectNodeDataMap(updatedResults, common, intg.IntegrationType, ctx) + } else { - extras := utils.ToMap[any](common) - extras["scan_type"] = intg.Resource + var updatedResults []map[string]interface{} + // inject node details to results + if integration.IsMessagingFormat(intg.IntegrationType) { + updatedResults = FormatForMessagingApps(results, intg.Resource) + } else { + updatedResults = []map[string]interface{}{} + for _, r := range results { + updatedResults = append(updatedResults, utils.ToMap[T](r)) + } + } - profileStart = time.Now() + updatedResults = injectNodeDataMap(updatedResults, common, intg.IntegrationType, ctx) - err = integrationModel.SendNotification(ctx, updatedResults, extras) - if err != nil { - updateIntegrationMetrics(ctx, intg.ID, - integrations.Metrics{Ok: 0, Error: int64(len(updatedResults))}) - return err - } else { - updateIntegrationMetrics(ctx, intg.ID, - integrations.Metrics{Ok: int64(len(updatedResults)), Error: 0}) - } + extras := utils.ToMap[any](common) + extras["scan_type"] = intg.Resource + // applicable only for scans + extras["scan_result_link"] = scanResultLink(consoleURL, scan.NodeType, intg.Resource, scan.ScanID) + extras["severity_counts"] = scan.SeverityCounts - totalSendTime = totalSendTime + time.Since(profileStart) + profileStart = time.Now() - log.Info().Msgf("Notification sent %s scan %d messages using %s id %d, time taken: %s", - intg.Resource, len(results), intg.IntegrationType, - intg.ID, time.Since(profileStart)) - } + err = integrationModel.SendNotification(ctx, updatedResults, extras) + if err != nil { + updateIntegrationMetrics(ctx, pgClient, intg.ID, + integrations.Metrics{Ok: 0, Error: int64(len(updatedResults)), IsSummary: false}) + return err + } else { + updateIntegrationMetrics(ctx, pgClient, intg.ID, + integrations.Metrics{Ok: int64(len(updatedResults)), Error: 0, IsSummary: false}) + } + totalSendTime += time.Since(profileStart) - if err := updateLastEventUpdatedAt(ctx, intg.ID, latestScanUpdatedAt); err != nil { - log.Error().Err(err).Msg("failed to to update last_event_updated_at") + log.Info().Msgf("Notification sent %s scan %d messages using %s id %d, time taken: %s", + intg.Resource, len(results), intg.IntegrationType, + intg.ID, time.Since(profileStart)) + } } log.Info().Msgf("%s Total Time taken for integration %s: %s", intg.Resource, intg.IntegrationType, time.Since(start)) - log.Debug().Msgf("Time taken for neo4j_query_nc: %s", neo4j_query_nc) - log.Debug().Msgf("Time taken for neo4j_query_c: %s", neo4j_query_c) - log.Debug().Msgf("Time taken for neo4j_query_default: %s", neo4j_query_default) + log.Debug().Msgf("Time taken for neo4j_query_nc: %s", neo4jQueryNc) + log.Debug().Msgf("Time taken for neo4j_query_c: %s", neo4jQueryC) + log.Debug().Msgf("Time taken for neo4j_query_default: %s", neo4jQueryDefault) log.Debug().Msgf("Time taken for neo4j_query_2: %s", totalQueryTime) log.Debug().Msgf("Time taken for sending data : %s", totalSendTime) + + // update the last event update_at + if err := updateLastEventUpdatedAt(ctx, pgClient, intg.ID, latestScanUpdatedAt); err != nil { + log.Error().Err(err).Msg("failed to to update last_event_updated_at") + } + return nil } -func updateLastEventUpdatedAt(ctx context.Context, intgId int32, updatedAt int64) error { - pgClient, err := directory.PostgresClient(ctx) - if err != nil { - log.Error().Err(err).Msg("unable to get sql client") - return err +func scanResultLink(consoleURL string, nodeType string, scanType string, scanID string) string { + + switch scanType { + + case "Vulnerability": + return fmt.Sprintf("%s/vulnerability/scan-results/%s", consoleURL, scanID) + + case "Secret": + return fmt.Sprintf("%s/secret/scan-results/%s", consoleURL, scanID) + + case "Malware": + return fmt.Sprintf("%s/malware/scan-results/%s", consoleURL, scanID) + + case "Compliance": + switch nodeType { + case "host": + return fmt.Sprintf("%s/posture/scan-results/linux/%s", consoleURL, scanID) + case "cluster": + return fmt.Sprintf("%s/posture/scan-results/kubernetes/%s", consoleURL, scanID) + default: + return fmt.Sprintf("%s/posture", consoleURL) + } + + case "CloudCompliance": + switch { + case strings.Contains(scanID, "aws"): + return fmt.Sprintf("%s/posture/cloud/scan-results/aws/%s", consoleURL, scanID) + case strings.Contains(scanID, "azure"): + return fmt.Sprintf("%s/posture/cloud/scan-results/azure/%s", consoleURL, scanID) + case strings.Contains(scanID, "gcp"): + return fmt.Sprintf("%s/posture/cloud/scan-results/gcp/%s", consoleURL, scanID) + default: + return fmt.Sprintf("%s/posture", consoleURL) + } + + default: + return consoleURL } +} + +func updateLastEventUpdatedAt(ctx context.Context, pgClient *postgresql_db.Queries, + intgID int32, updatedAt int64) error { last := postgresql_db.UpdateIntegrationLastEventUpdatedAtParams{ - ID: intgId, + ID: intgID, LastEventUpdatedAt: sql.NullTime{ Time: time.UnixMilli(updatedAt), Valid: true, @@ -485,12 +565,8 @@ func updateLastEventUpdatedAt(ctx context.Context, intgId int32, updatedAt int64 return nil } -func updateIntegrationMetrics(ctx context.Context, intgID int32, metrics integrations.Metrics) error { - pgClient, err := directory.PostgresClient(ctx) - if err != nil { - log.Error().Err(err).Msg("unable to get sql client") - return err - } +func updateIntegrationMetrics(ctx context.Context, pgClient *postgresql_db.Queries, + intgID int32, metrics integrations.Metrics) { param := postgresql_db.SetIntegrationMetricsParams{ID: intgID} @@ -504,10 +580,9 @@ func updateIntegrationMetrics(ctx context.Context, intgID int32, metrics integra if err := pgClient.SetIntegrationMetrics(ctx, param); err != nil { log.Error().Err(err).Msg("failed to update last event updated at") - return err + return } - return nil } func FormatForMessagingApps[T any](results []T, resourceType string) []map[string]interface{} { @@ -590,7 +665,6 @@ func injectNodeDataMap(results []map[string]interface{}, common model.ScanResult flag := integration.IsMessagingFormat(integrationType) for _, r := range results { - //m := utils.ToMap[T](r) r["node_id"] = common.NodeID r["scan_id"] = common.ScanID r["node_name"] = common.NodeName diff --git a/deepfence_worker/cronscheduler/init_neo4j.go b/deepfence_worker/cronscheduler/init_neo4j.go index 8ab4d3c44b..28f5fad942 100644 --- a/deepfence_worker/cronscheduler/init_neo4j.go +++ b/deepfence_worker/cronscheduler/init_neo4j.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/deepfence/ThreatMapper/deepfence_server/model" "github.com/deepfence/ThreatMapper/deepfence_utils/directory" "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/telemetry" @@ -92,6 +93,8 @@ func initNeo4jDatabase(ctx context.Context) error { //Set the base updated_at field on the ALIAS relationship RunDisplayError(ctx, session, "WITH TIMESTAMP() as T MATCH(:ContainerImage) -[a:ALIAS]-> (:ImageTag) WHERE a.updated_at IS NULL SET a.updated_at=T") + RunDisplayError(ctx, session, "MATCH (n:CloudNode) WHERE COALESCE(n.refresh_status, '') = '' AND n.cloud_provider in ['"+model.PostureProviderAWS+"','"+model.PostureProviderGCP+"','"+model.PostureProviderAzure+"'] SET n.refresh_status='"+utils.ScanStatusStarting+"', n.refresh_message=''") + return nil } diff --git a/deepfence_worker/go.mod b/deepfence_worker/go.mod index 847e55049a..bb4973975a 100644 --- a/deepfence_worker/go.mod +++ b/deepfence_worker/go.mod @@ -34,7 +34,7 @@ require ( github.com/deepfence/YaraHunter v0.0.0-00010101000000-000000000000 github.com/deepfence/agent-plugins-grpc v1.1.0 github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20240626143546-e4ec9311fdf9 - github.com/deepfence/match-scanner v0.0.0-20240627065846-d2405fb72cfb + github.com/deepfence/match-scanner v0.0.0-20240701181002-ece6f13f296f github.com/deepfence/package-scanner v0.0.0-00010101000000-000000000000 github.com/go-chi/chi/v5 v5.0.8 github.com/hibiken/asynq v0.24.1 diff --git a/deepfence_worker/go.sum b/deepfence_worker/go.sum index e58fad65ab..d9094e36b5 100644 --- a/deepfence_worker/go.sum +++ b/deepfence_worker/go.sum @@ -212,8 +212,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= -github.com/deepfence/match-scanner v0.0.0-20240627065846-d2405fb72cfb h1:E3ffVItZVnhj1CD6UO/FPKyPz7Osinc4M770Jmm4JKc= -github.com/deepfence/match-scanner v0.0.0-20240627065846-d2405fb72cfb/go.mod h1:eSZaZ9yVo4FoA3hJzTVWJL8HKvm3YPQzw3Nx/NKuq9A= +github.com/deepfence/match-scanner v0.0.0-20240701181002-ece6f13f296f h1:TqJx9TO/YD/W6CgDb1f4IkZqRAM31i6O0I2OnrE6xCI= +github.com/deepfence/match-scanner v0.0.0-20240701181002-ece6f13f296f/go.mod h1:eSZaZ9yVo4FoA3hJzTVWJL8HKvm3YPQzw3Nx/NKuq9A= github.com/deepfence/vessel v0.12.3 h1:C34t+sV+JoFdfYhg+uyS+YOEDAFIYjBKHShD3emDISA= github.com/deepfence/vessel v0.12.3/go.mod h1:bY97YUKMm0Oxasz/9o7Te60FjWCQWUYpgiWNC1E00xo= github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4= diff --git a/deepfence_worker/ingesters/cloud_resource.go b/deepfence_worker/ingesters/cloud_resource.go index 035d58fbee..4b28313927 100644 --- a/deepfence_worker/ingesters/cloud_resource.go +++ b/deepfence_worker/ingesters/cloud_resource.go @@ -4,12 +4,12 @@ import ( "context" "encoding/json" "fmt" + "sort" "time" "github.com/deepfence/ThreatMapper/deepfence_utils/directory" "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/telemetry" - "github.com/deepfence/ThreatMapper/deepfence_utils/utils" ingestersUtil "github.com/deepfence/ThreatMapper/deepfence_utils/utils/ingesters" "github.com/neo4j/neo4j-go-driver/v5/neo4j" ) @@ -266,7 +266,8 @@ func CommitFuncCloudResourceRefreshStatus(ctx context.Context, ns string, cs []i UNWIND $batch as row MATCH (n:CloudNode{node_id: row.cloud_node_id}) SET n.refresh_status = row.refresh_status, - n.refresh_message = row.refresh_message`, + n.refresh_message = row.refresh_message, + n.refresh_updated_at = row.updated_at`, map[string]interface{}{ "batch": ResourceRefreshStatusToMaps(cs), }, @@ -276,42 +277,14 @@ func CommitFuncCloudResourceRefreshStatus(ctx context.Context, ns string, cs []i } func ResourceRefreshStatusToMaps(data []ingestersUtil.CloudResourceRefreshStatus) []map[string]interface{} { - statusBuff := map[string]map[string]interface{}{} - for _, i := range data { - statusMap := i.ToMap() + statuses := make([]map[string]interface{}, len(data)) - cloudNodeId, ok := statusMap["cloud_node_id"].(string) - if !ok { - log.Error().Msgf("failed to convert cloud_node_id to string, data: %v", statusMap) - continue - } - - newStatus, ok := statusMap["refresh_status"].(string) - if !ok { - log.Error().Msgf("failed to convert refresh_status to string, data: %v", statusMap) - continue - } + sort.Slice(data, func(i, j int) bool { + return data[i].UpdatedAt < data[j].UpdatedAt + }) - old, found := statusBuff[cloudNodeId] - if !found { - statusBuff[cloudNodeId] = statusMap - } else { - oldStatus, ok := old["refresh_status"].(string) - if !ok { - log.Error().Msgf("failed to convert refresh_status to string, data: %v", old) - continue - } - if newStatus != oldStatus { - if newStatus == utils.ScanStatusSuccess || - newStatus == utils.ScanStatusFailed || newStatus == utils.ScanStatusCancelled { - statusBuff[cloudNodeId] = statusMap - } - } - } - } - statuses := []map[string]interface{}{} - for _, v := range statusBuff { - statuses = append(statuses, v) + for i, d := range data { + statuses[i] = d.ToMap() } return statuses } diff --git a/deepfence_worker/metrics.go b/deepfence_worker/metrics.go index 578583f068..47309a86ce 100644 --- a/deepfence_worker/metrics.go +++ b/deepfence_worker/metrics.go @@ -203,7 +203,7 @@ func newIntegrationCollector() *IntegrationCollector { integration: prometheus.NewDesc( "integration_metrics", "integration by type and metrics", - []string{"namespace", "id", "type", "status"}, nil, + []string{"namespace", "id", "type", "status", "summary"}, nil, ), } } @@ -237,9 +237,11 @@ func (collector *IntegrationCollector) Collect(ch chan<- prometheus.Metric) { for _, intg := range integrations { ch <- prometheus.MustNewConstMetric(collector.integration, prometheus.CounterValue, - float64(intg.Metrics.Ok), ns, strconv.Itoa(int(intg.ID)), intg.IntegrationType, "ok") + float64(intg.Metrics.Ok), ns, strconv.Itoa(int(intg.ID)), + intg.IntegrationType, "ok", strconv.FormatBool(intg.Metrics.IsSummary)) ch <- prometheus.MustNewConstMetric(collector.integration, prometheus.CounterValue, - float64(intg.Metrics.Error), ns, strconv.Itoa(int(intg.ID)), intg.IntegrationType, "error") + float64(intg.Metrics.Error), ns, strconv.Itoa(int(intg.ID)), + intg.IntegrationType, "error", strconv.FormatBool(intg.Metrics.IsSummary)) } } diff --git a/deepfence_worker/tasks/malwarescan/malwarescan.go b/deepfence_worker/tasks/malwarescan/malwarescan.go index 0e8089bd25..c962298179 100644 --- a/deepfence_worker/tasks/malwarescan/malwarescan.go +++ b/deepfence_worker/tasks/malwarescan/malwarescan.go @@ -33,7 +33,7 @@ var ( failOnCompileWarning = false malwareRulesDir = "/usr/local/malware" malwareRulesPath = "/usr/local/malware/yara-rules" - malwareConfigPath = "/malware-config" + malwareConfigPath = "/malware-config/config.yaml" opts *malwareConfig.Options yaraconfig config.Config yr *yararules.YaraRules @@ -77,7 +77,10 @@ func checkMalwareRulesUpdate(ctx context.Context) error { if err := workerUtils.UpdateRules(ctx, path, malwareRulesDir); err != nil { return err } - opts, yaraconfig, yr = initMalwareScanner() + opts, yaraconfig, yr, err = initMalwareScanner() + if err != nil { + return err + } } return nil @@ -242,6 +245,7 @@ func (s MalwareScan) StartMalwareScan(ctx context.Context, task *asynq.Task) err err = scanCtx.Checkpoint("After skopeo download") if err != nil { + log.Error().Msg(err.Error()) return err } // set imageName in malwareScanner @@ -251,8 +255,10 @@ func (s MalwareScan) StartMalwareScan(ctx context.Context, task *asynq.Task) err err = malwareScanner.Scan(scanCtx, malwareScan.ImageScan, "", imageName, params.ScanID, func(i output.IOCFound, s string) { + scanResult = append(scanResult, i) }) if err != nil { + log.Error().Msg(err.Error()) return err } @@ -280,7 +286,7 @@ func (s MalwareScan) StartMalwareScan(ctx context.Context, task *asynq.Task) err return nil } -func initMalwareScanner() (*malwareConfig.Options, config.Config, *yararules.YaraRules) { +func initMalwareScanner() (*malwareConfig.Options, config.Config, *yararules.YaraRules, error) { opts := malwareConfig.NewDefaultOptions() opts.RulesPath = &malwareRulesPath opts.ConfigPath = &malwareConfigPath @@ -289,13 +295,15 @@ func initMalwareScanner() (*malwareConfig.Options, config.Config, *yararules.Yar yaraconfig, err := config.ParseConfig(*opts.ConfigPath) if err != nil { log.Error().Msg(err.Error()) + return nil, yaraconfig, nil, err } yr := yararules.New(*opts.RulesPath) err = yr.Compile(malwareScanConstants.Filescan, *opts.FailOnCompileWarning) if err != nil { log.Error().Msg(err.Error()) + return nil, yaraconfig, nil, err } - return opts, yaraconfig, yr + return opts, yaraconfig, yr, nil } diff --git a/docs/docs/cloudscanner/aws.md b/docs/docs/cloudscanner/aws.md index 62bc46a7d7..d4f1e78d50 100644 --- a/docs/docs/cloudscanner/aws.md +++ b/docs/docs/cloudscanner/aws.md @@ -81,6 +81,8 @@ module "deepfence-cloud-scanner_example_single-account" { mgmt-console-url = "" mgmt-console-port = "443" deepfence-key = "" + # AWS Account Name (Optional, for easy identification) + account_name = "" image = "quay.io/deepfenceio/cloud_scanner_ce:THREATMAPPER_VERSION" # Task CPU Units (Default: 4 vCPU) cpu = "4096" diff --git a/docs/docs/cloudscanner/gcp.md b/docs/docs/cloudscanner/gcp.md index ff01921cfa..6761e50d12 100644 --- a/docs/docs/cloudscanner/gcp.md +++ b/docs/docs/cloudscanner/gcp.md @@ -21,6 +21,8 @@ module "cloud-scanner_example_single-project" { mgmt-console-url = "" mgmt-console-port = "443" deepfence-key = "" + # GCP Project Name (Optional, for easy identification) + project_name = "" image_name = "us-east1-docker.pkg.dev/deepfenceio/deepfence/cloud_scanner_ce:THREATMAPPER_VERSION" # project_id example: dev1-123456 project_id = "" diff --git a/docs/vulnerability_feeds/listing.json b/docs/vulnerability_feeds/listing.json index 1212b5b831..170e6a7a79 100644 --- a/docs/vulnerability_feeds/listing.json +++ b/docs/vulnerability_feeds/listing.json @@ -2,54 +2,54 @@ "available": { "3": [ { - "built": "2024-06-28T00:14:41.212177109Z", + "built": "2024-07-02T00:13:36.994488193Z", "version": 3, - "url": "https://threat-intel.deepfence.io/vulnerability-db/releases/download/threatintel-vuln-v3-2024-06-28_00-02-47/threatintel-vuln-v3-2024-06-28_00-02-47.tar.gz", - "checksum": "6e007c26222d13214a2f9ed84a1b6168d7f220b58b3abdef1275097b10063134" + "url": "https://threat-intel.deepfence.io/vulnerability-db/releases/download/threatintel-vuln-v3-2024-07-02_00-02-33/threatintel-vuln-v3-2024-07-02_00-02-33.tar.gz", + "checksum": "ec15b20efc33447db14224822a95e349b84b7f6cc1fbc27abfbc3467afa108f9" }, { - "built": "2024-06-29T00:13:54.883824223Z", + "built": "2024-07-03T00:13:43.763310651Z", "version": 3, - "url": "https://threat-intel.deepfence.io/vulnerability-db/releases/download/threatintel-vuln-v3-2024-06-29_00-02-25/threatintel-vuln-v3-2024-06-29_00-02-25.tar.gz", - "checksum": "b1c02f19752fe2b9edbcdb9676a5a51caf856e891b4af4414a4227dc3821794f" + "url": "https://threat-intel.deepfence.io/vulnerability-db/releases/download/threatintel-vuln-v3-2024-07-03_00-02-40/threatintel-vuln-v3-2024-07-03_00-02-40.tar.gz", + "checksum": "f68bd1c7805c7c96d1556168c691d26a1c0bd13eac5fe00f7c385155311ca419" }, { - "built": "2024-06-30T00:13:55.172505189Z", + "built": "2024-07-04T00:14:41.155820756Z", "version": 3, - "url": "https://threat-intel.deepfence.io/vulnerability-db/releases/download/threatintel-vuln-v3-2024-06-30_00-02-27/threatintel-vuln-v3-2024-06-30_00-02-27.tar.gz", - "checksum": "2702efb332c2c1c45b634737e2422a7dc5c1e0a13402c7a7601e68a13228941b" + "url": "https://threat-intel.deepfence.io/vulnerability-db/releases/download/threatintel-vuln-v3-2024-07-04_00-02-19/threatintel-vuln-v3-2024-07-04_00-02-19.tar.gz", + "checksum": "8cca4816dd64be972f2f5721be61c5d755a02883b1c1cf31c6d58defac2d32d6" }, { - "built": "2024-07-01T00:13:40.377637385Z", + "built": "2024-07-05T00:13:51.524295992Z", "version": 3, - "url": "https://threat-intel.deepfence.io/vulnerability-db/releases/download/threatintel-vuln-v3-2024-07-01_00-02-44/threatintel-vuln-v3-2024-07-01_00-02-44.tar.gz", - "checksum": "bc7a35c4714fe713244a7109a1434dccae7f597278cf9c7336fa8e11b0561140" + "url": "https://threat-intel.deepfence.io/vulnerability-db/releases/download/threatintel-vuln-v3-2024-07-05_00-02-39/threatintel-vuln-v3-2024-07-05_00-02-39.tar.gz", + "checksum": "d965859b48c6f9c395f724c3275a35ce65c5eac37d0d9a70aafe195ea240872f" } ], "5": [ { - "built": "2024-06-29T14:36:08.794072895Z", + "built": "2024-07-03T14:26:24.407210575Z", "version": 5, - "url": "https://threat-intel.deepfence.io/vulnerability-db/releases/download/threatintel-vuln-v5-2024-06-29_13-24-17/threatintel-vuln-v5-2024-06-29_13-24-17.tar.gz", - "checksum": "c48957279cfbdd354b0e13b7c5843547b00cbc87618a7a468c2b4730b585f651" + "url": "https://threat-intel.deepfence.io/vulnerability-db/releases/download/threatintel-vuln-v5-2024-07-03_13-20-16/threatintel-vuln-v5-2024-07-03_13-20-16.tar.gz", + "checksum": "5449823099862fe3f8463bb02281b8991a07201e3cdfc277f6c6f43c5fd118b4" }, { - "built": "2024-06-30T03:00:59.325567564Z", + "built": "2024-07-04T02:57:44.20632311Z", "version": 5, - "url": "https://threat-intel.deepfence.io/vulnerability-db/releases/download/threatintel-vuln-v5-2024-06-30_01-47-41/threatintel-vuln-v5-2024-06-30_01-47-41.tar.gz", - "checksum": "25b7e7cd74e8ab3f31182fe894b8fd3d7d725d96504c33dff779c70ac8b4b7b5" + "url": "https://threat-intel.deepfence.io/vulnerability-db/releases/download/threatintel-vuln-v5-2024-07-04_01-47-42/threatintel-vuln-v5-2024-07-04_01-47-42.tar.gz", + "checksum": "f2d48e9c6f40d4fc1401e3c7ca68d566569286cc31ccf6e068c888d831afd20a" }, { - "built": "2024-06-30T14:31:16.209415361Z", + "built": "2024-07-04T14:32:02.795503284Z", "version": 5, - "url": "https://threat-intel.deepfence.io/vulnerability-db/releases/download/threatintel-vuln-v5-2024-06-30_13-20-18/threatintel-vuln-v5-2024-06-30_13-20-18.tar.gz", - "checksum": "6cb13fc86db44449a5f1fff895de4a1cac2a1095c1ff8e266969b68db3a5777e" + "url": "https://threat-intel.deepfence.io/vulnerability-db/releases/download/threatintel-vuln-v5-2024-07-04_13-23-21/threatintel-vuln-v5-2024-07-04_13-23-21.tar.gz", + "checksum": "60bc994283b75a45d27f2d3387e41b76224b3409491948a214fd825e7c0fdf7c" }, { - "built": "2024-07-01T02:48:57.308556995Z", + "built": "2024-07-05T02:51:35.567140143Z", "version": 5, - "url": "https://threat-intel.deepfence.io/vulnerability-db/releases/download/threatintel-vuln-v5-2024-07-01_01-45-26/threatintel-vuln-v5-2024-07-01_01-45-26.tar.gz", - "checksum": "bacff416d5c58fb8cb1e365c5d803dbbd133db69ccc57566a70ba60bf2577eca" + "url": "https://threat-intel.deepfence.io/vulnerability-db/releases/download/threatintel-vuln-v5-2024-07-05_01-46-44/threatintel-vuln-v5-2024-07-05_01-46-44.tar.gz", + "checksum": "30797ed9f9e9c38c729c27ca2bd930e78a5af0259cc48a4d3aab909319d7f090" } ] } diff --git a/golang_deepfence_sdk b/golang_deepfence_sdk index 226fd1e1e6..e4ec9311fd 160000 --- a/golang_deepfence_sdk +++ b/golang_deepfence_sdk @@ -1 +1 @@ -Subproject commit 226fd1e1e6bb7b9f4e179d6f738bb7cd9c55dd72 +Subproject commit e4ec9311fdf96022b9a0fad59e63e530ba50587f