diff --git a/README.md b/README.md index 9240caa0..6ba1e16b 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ The dashboard is broken into the five sections and each pillar is laid out in a ## Detection example -Security Analysis Tool (SAT) analyzes 37 best practices, with more on the way. In the example below, the SAT scan highlights one finding that surfaces a potential risk, and one that meets Databricks' best practices. The Deprecated runtime versions check is red indicating that there are runtimes that are deprecated. Workloads on unsupported runtime versions may continue to run, but they receive no Databricks support or fixes. The Remediation column in the screenshot describes the risk and links to the documentation of the Databricks runtime versions that are currently supported. +Security Analysis Tool (SAT) analyzes 60 best practices, with more on the way. In the example below, the SAT scan highlights one finding that surfaces a potential risk, and one that meets Databricks' best practices. The Deprecated runtime versions check is red indicating that there are runtimes that are deprecated. Workloads on unsupported runtime versions may continue to run, but they receive no Databricks support or fixes. The Remediation column in the screenshot describes the risk and links to the documentation of the Databricks runtime versions that are currently supported. On the other hand, the Log delivery check is green, confirming that the workspace follows Databricks security best practices. Run these checks regularly to comprehensively view Databricks account workspace security and ensure continuous improvement. diff --git a/configs/security_best_practices.csv b/configs/security_best_practices.csv index 4d6e845c..b5bcc65b 100644 --- a/configs/security_best_practices.csv +++ b/configs/security_best_practices.csv @@ -1,53 +1,63 @@ -id,check_id,category,check,evaluation_value,severity,recommendation,aws,azure,gcp,enable,alert,logic,api,aws_doc_url,azure_doc_url,gcp_doc_url -1,DP-1,Data Protection,Secrets management,1,Medium,Store and use secrets securely,1,1,1,1,0,Check if there are any secrets configured in the workspace by listing scopes and looking for the secrets count against the configured baseline value in those scopes,curl --netrc -X GET \ https:///api/2.0/secrets/scopes/list,https://docs.databricks.com/security/secrets/index.html,https://learn.microsoft.com/en-us/azure/databricks/security/secrets/,https://docs.gcp.databricks.com/security/secrets/index.html -2,DP-2,Data Protection,Cluster encryption,-1,Low,All clusters should enable local disk encryption,1,1,0,1,0,Check if enable_local_disk_encryption is set as false for any cluster,curl --netrc -X GET \ https:///api/2.0/clusters/list,https://docs.databricks.com/clusters/configure.html#local-disk-encryption,https://learn.microsoft.com/en-us/azure/databricks/clusters/configure#--local-disk-encryption,https://docs.gcp.databricks.com/clusters/configure.html#local-disk-encryption -3,DP-3,Data Protection,BYOK ,-1,Low,Add a customer-managed key for managed services and workspace storage,1,0,0,1,0,Check if storage_customer_managed_key_id and managed_services_customer_managed_key_id are set for each workspace,curl -n -X GET 'https://accounts.cloud.databricks.com/api/2.0/accounts//workspaces',https://docs.databricks.com/security/keys/customer-managed-keys-storage-aws.html,https://learn.microsoft.com/en-us/azure/databricks/security/keys/customer-managed-keys,N/A -4,DP-4,Data Protection,Object storage encryption,-1,High,Encrypt object storage data buckets and restrict access,1,1,1,1,0,Manual check (Update cofiguration status using /notebooks/Setup/8. update_workspace_configuration notebook),Manual check,https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucket-encryption.html,https://learn.microsoft.com/en-us/azure/storage/common/storage-service-encryption,https://cloud.google.com/storage/docs/encryption -5,DP-5,Data Protection,Results downloading,-1,High,Disable download button for notebook results,1,1,1,1,0,Check workspace-conf for enableResultsDownloading setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enableResultsDownloading',https://docs.databricks.com/administration-guide/workspace/notebooks.html#manage-download-results,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/workspace/notebooks#manage-download-results,https://docs.gcp.databricks.com/administration-guide/workspace/notebooks.html#manage-download-results -6,GOV-1,Governance,Cluster policies,-1,High,Configure cluster policies to enforce data access patterns and control costs,1,1,1,1,0,Check if policy_id is set for clusters,curl --netrc -X GET \ https:///api/2.0/clusters/list \ | jq .,https://docs.databricks.com/administration-guide/clusters/policies.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/clusters/policies,https://docs.gcp.databricks.com/administration-guide/clusters/policies.html -7,GOV-2,Governance,PAT Tokens about to expire,7,High,"Set lifetime limit, but also regularly review PAT tokens to avoid expired tokens impacting authentications",1,1,1,1,0,Check each token expiry_time and report if the expiry_time is within configured days,curl --netrc -X GET \ https:///api/2.0/token/list | jq .,https://docs.databricks.com/administration-guide/access-control/tokens.html#manage-personal-access-tokens,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/access-control/tokens#manage-personal-access-tokens,https://docs.gcp.databricks.com/administration-guide/access-control/tokens.html#manage-personal-access-tokens -8,GOV-3,Governance,Log delivery configurations,-1,High,Configure Databricks audit log delivery,1,1,1,1,0,"Check account log-delivery configuration and look for audit log config with log_type set as ""AUDIT_LOGS"" and status set as ""ENABLED""",curl -netrc -X GET \ 'https://accounts.cloud.databricks.com/api/2.0/accounts//log-delivery',https://docs.databricks.com/administration-guide/account-settings/audit-logs.html#configure-audit-log-delivery,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/account-settings/audit-logs#configure-audit-log-delivery,https://docs.gcp.databricks.com/administration-guide/account-settings/audit-logs.html#configure-audit-log-delivery -9,GOV-4,Governance,Long running clusters,24,Medium,Automatically restart long running clusters,1,1,1,1,0,Check each running cluster's last restart time till now and report on clusters that were running longer than the configured number of days without a restart,curl --netrc -X GET \ https:///api/2.0/clusters/list \ | jq .,https://docs.databricks.com/clusters/clusters-manage.html#restart-a-cluster-to-update-it-with-the-latest-images,https://learn.microsoft.com/en-us/azure/databricks/clusters/clusters-manage#--restart-a-cluster-to-update-it-with-the-latest-images,https://docs.gcp.databricks.com/clusters/clusters-manage.html#restart-a-cluster-to-update-it-with-the-latest-images -10,GOV-5,Governance,Deprecated runtime versions,-1,High,Deprecated runtime version detected. Please update your cluster runtimes to Databricks supported runtimes,1,1,1,1,0,List clusters with spark version that is not in the supported spark versions,curl --netrc -X GET \ https:///api/2.0/clusters/spark-versions,https://docs.databricks.com/release-notes/runtime/releases.html,https://learn.microsoft.com/en-us/azure/databricks/release-notes/runtime/releases,https://docs.gcp.databricks.com/release-notes/runtime/releases.html -11,GOV-6,Governance,All Purpose Cluster custom tags,-1,Low,Configure cluster tagging to monitor usage and enable charge-back,1,1,1,1,0,check if custom_tags is set for clusters,curl --netrc -X GET \ https:///api/2.0/clusters/list \ | jq .,https://docs.databricks.com/administration-guide/account-settings/usage-detail-tags-aws.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/account-settings/usage-detail-tags-azure,https://docs.gcp.databricks.com/administration-guide/account-settings-gcp/usage-detail-tags-gcp.html -12,GOV-7,Governance,Job Cluster custom tags,-1,Low,Configure job tagging to monitor usage and enable charge-back,1,1,1,1,0,Check if settings.new_cluster.custom_tags is not null for job clusters,curl --netrc -X GET \ https:///api/2.0/jobs/list \ | jq,https://docs.databricks.com/administration-guide/account-settings/usage-detail-tags-aws.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/account-settings/usage-detail-tags-azure,https://docs.gcp.databricks.com/administration-guide/account-settings-gcp/usage-detail-tags-gcp.html -13,GOV-8,Governance,All Purpose Cluster log configuration,-1,Low,Configure Databricks cluster log delivery,1,1,1,1,0,Check if cluster_log_conf is set for clusters,curl --netrc -X GET \ https:///api/2.0/clusters/list \ | jq .,https://docs.databricks.com/clusters/configure.html#cluster-log-delivery-1,https://learn.microsoft.com/en-us/azure/databricks/clusters/configure#cluster-log-delivery,https://docs.gcp.databricks.com/clusters/configure.html#cluster-log-delivery-1 -14,GOV-9,Governance,Job Cluster log configuration,-1,Low,Configure Databricks job custer log delivery,1,1,1,1,0,Check if cluster_log_conf is set for job clusters,curl --netrc -X GET \ https:///api/2.0/jobs/list \ | jq,https://docs.databricks.com/clusters/configure.html#cluster-log-delivery-1,https://learn.microsoft.com/en-us/azure/databricks/clusters/configure#cluster-log-delivery,https://docs.gcp.databricks.com/clusters/configure.html#cluster-log-delivery-1 -15,GOV-10,Governance,Managed Tables,1,Low,The DBFS root is not intended for production customer data,1,1,1,1,0,Check the /user/hive/warehouse/ folder for any data folders stored more than the configured value,"curl --netrc -X GET \ https:///api/2.0/dbfs/list \ --data '{ ""path"": ""/user/hive/warehouse/"" }'",https://docs.databricks.com/data/databricks-file-system.html#configuration-and-usage-recommendations,https://learn.microsoft.com/en-us/azure/databricks/data/databricks-file-system#configuration-and-usage-recommendations,https://docs.gcp.databricks.com/dbfs/index.html#configuration-and-usage-recommendations -16,GOV-11,Governance,Mounts,1,Low,Avoid FUSE mounts for accessing production data,1,1,1,1,0,Check for mnt paths in dbutils.fs.mounts() and report if there are datasources loaded as FUSE mounts to the workspace than the configured value,dbutils.fs.mounts(),https://docs.databricks.com/data/databricks-file-system.html#configuration-and-usage-recommendations,https://learn.microsoft.com/en-us/azure/databricks/data/databricks-file-system#configuration-and-usage-recommendations,https://docs.gcp.databricks.com/dbfs/index.html#configuration-and-usage-recommendations -17,GOV-12,Governance,UC enabled clusters,-1,High,Use UC enabled clusters,1,1,0,1,0,Check if there are clusters without data_security_mode as (USER_ISOLATION or SINGLE_USER) or data_security_mode as none,curl --netrc -X GET \ https:///api/2.0/clusters/list \ | jq .,https://docs.databricks.com/data-governance/unity-catalog/index.html#cluster-access-modes-for-unity-catalog,https://learn.microsoft.com/en-us/azure/databricks/data-governance/unity-catalog/#cluster-access-modes-for-unity-catalog,https://docs.gcp.databricks.com/data-governance/unity-catalog/index.html#cluster-access-modes-for-unity-catalog -18,IA-1,Identity & Access,SSO,-1,High,Authenticate via single sign-on and leverage multi-factor authentication,1,1,1,1,0,Manual check (Update cofiguration status using /notebooks/Setup/8. update_workspace_configuration notebook),Manual check,https://docs.databricks.com/administration-guide/users-groups/single-sign-on/index.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/users-groups/single-sign-on/,https://docs.gcp.databricks.com/administration-guide/users-groups/single-sign-on/index.html -19,IA-2,Identity & Access,SCIM,-1,High,Keep an up-to-date user list by using SCIM,1,1,1,1,0,Manual check (Update cofiguration status using /notebooks/Setup/8. update_workspace_configuration notebook),Manual check,https://docs.databricks.com/administration-guide/users-groups/scim/index.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/users-groups/scim/index,https://docs.gcp.databricks.com/administration-guide/users-groups/scim/index.html -20,IA-3,Identity & Access,Table Access Control,-1,High,Enable Table Access Control in admin settings so that you can utilize Table ACL clusters that enforce user isolation,1,1,1,1,0,Manual check (Update cofiguration status using /notebooks/Setup/8. update_workspace_configuration notebook),Manual check,https://docs.databricks.com/administration-guide/access-control/table-acl.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/access-control/table-acl,https://docs.gcp.databricks.com/administration-guide/access-control/table-acl.html -21,IA-4,Identity & Access,PAT Tokens with no lifetime limit,-1,Medium,Configure maximum token lifetimes for future tokens using token management,1,1,1,1,0,Check each token expiry_time and report if the expiry_time is not within the configured number of days or set to never expire by using -1 as the value,curl --netrc -X GET \ https:///api/2.0/token/list,https://docs.databricks.com/administration-guide/access-control/tokens.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/access-control/tokens#manage-personal-access-tokens,https://docs.gcp.databricks.com/administration-guide/access-control/tokens.html -22,INFO-1,Informational,Instance Pool custom tag,-1,Low,Configure tagging to monitor usage and enable charge-back,1,1,1,1,0,Check if instance-pools have custom tags configured,curl --netrc -X GET \ https:///api/2.0/instance-pools/list | jq .,https://docs.databricks.com/clusters/instance-pools/configure.html#pool-tags,https://learn.microsoft.com/en-us/azure/databricks/clusters/instance-pools/configure#pool-tags,https://docs.gcp.databricks.com/clusters/instance-pools/configure.html#pool-tags -23,INFO-2,Informational,Max concurrent runs,5,Low,Limit the number of parallel runs for a given job to avoid resource contention,1,1,1,1,0,Check if max_concurrent_runs configuration for each job is less than configured value,curl --netrc -X GET \ https:///api/2.0/jobs/list \ | jq,https://docs.databricks.com/data-engineering/jobs/jobs.html#max-concurrent-runs,https://learn.microsoft.com/en-us/azure/databricks/data-engineering/jobs/jobs#max-concurrent-runs,https://docs.gcp.databricks.com/data-engineering/jobs/jobs.html#max-concurrent-runs -24,INFO-3,Informational,Global libraries,-1,Low,"Global libraries are discouraged for security reasons. Use cluster libraries or notebook-scoped libraries to improve startup time for clusters that don't require that library, and to improve flexibility where that library is not required.",1,1,1,1,0,Check if is_library_for_all_clusters is set as true for any library that is configured for clusters,curl --netrc -X GET \ https:///api/2.0/libraries/all-cluster-statuses | jq .,https://docs.databricks.com/libraries/cluster-libraries.html,https://learn.microsoft.com/en-us/azure/databricks/libraries/cluster-libraries,https://docs.gcp.databricks.com/libraries/cluster-libraries.html -25,INFO-4,Informational,User Privileges,5,Low,Limit number of users with cluster create privileges,1,1,1,1,0,"Check entitlements.value for each user and look if they have 'allow-cluster-create', 'allow-instance-pool-create' as permission, and count the number of users with such permissions to report if that crossed the configured threshold.",curl --netrc -X GET \ https:///api/2.0/preview/scim/v2/Users,https://docs.databricks.com/security/access-control/cluster-acl.html,https://learn.microsoft.com/en-us/azure/databricks/security/access-control/cluster-acl,https://docs.gcp.databricks.com/security/access-control/cluster-acl.html -26,INFO-5,Informational,Global Init Script,-1,Medium,"Global init scripts are discouraged for security, improved startup and flexibility reasons",1,1,1,1,0,Check if there are any global-init-scripts configured,curl --netrc -X GET \ https:///api/2.0/global-init-scripts,https://docs.databricks.com/clusters/init-scripts.html#global-init-scripts,https://learn.microsoft.com/en-us/azure/databricks/clusters/init-scripts#global-init-scripts,https://docs.gcp.databricks.com/clusters/init-scripts.html#global-init-scripts -27,INFO-6,Informational,Admin Count,2,Low,Limit the number of admin accounts so that most users are not admins,1,1,1,1,0,"Check members count of diplayName ""admins"" in groups to report if the count is more than the configured threshold",curl --netrc -X GET \ https:///api/2.0/preview/scim/v2/Groups,https://docs.databricks.com/administration-guide/users-groups/users.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/users-groups/users,https://docs.gcp.databricks.com/administration-guide/users-groups/users.html -28,INFO-7,Informational,VPC (or VNET) Peering,-1,Medium,VPC (or VNET) endpoints or Private Link are recommended,1,1,1,1,0,Manual check (Update cofiguration status using /notebooks/Setup/8. update_workspace_configuration notebook),Manual check,https://docs.aws.amazon.com/vpc/latest/privatelink/create-interface-endpoint.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/cloud-configurations/azure/vnet-inject,https://cloud.google.com/vpc/docs/private-access-options -29,INFO-8,Informational,Job View Acls,-1,High,Use Job Visibility Control to prevent users from viewing jobs where they do not have permissions,1,1,1,1,0,Check workspace-conf for enableJobViewAcls setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enableJobViewAcls',https://docs.databricks.com/administration-guide/access-control/jobs-acl.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/access-control/jobs-acl,https://docs.gcp.databricks.com/administration-guide/access-control/jobs-acl.html -30,INFO-9,Informational,Cluster View Acls,-1,High,Use Cluster Visibility Control to prevent users from viewing clusters where they do not have permissions,1,1,1,1,0,Check workspace-conf for enforceClusterViewAcls setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enforceClusterViewAcls',https://docs.databricks.com/security/access-control/cluster-acl.html,https://learn.microsoft.com/en-us/azure/databricks/security/access-control/cluster-acl,https://docs.gcp.databricks.com/security/access-control/cluster-acl.html -31,INFO-10,Informational,Workspace View Acls,-1,High,Use Workspace Visibility Control to prevent users from viewing notebooks and other objects where they do not have permissions in the workspace,1,1,1,1,0,Check workspace-conf for enforceWorkspaceViewAcls setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enforceWorkspaceViewAcls',https://docs.databricks.com/security/access-control/workspace-acl.html,https://learn.microsoft.com/en-us/azure/databricks/security/access-control/workspace-acl,https://docs.gcp.databricks.com/security/access-control/workspace-acl.html -32,INFO-11,Informational,Project Type In Workspace,-1,High,It is recommended to store code in Git repos,1,1,1,1,0,Check workspace-conf for enableProjectTypeInWorkspace setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enableProjectTypeInWorkspace',https://docs.databricks.com/repos/index.html,https://learn.microsoft.com/en-us/azure/databricks/repos/index,https://docs.gcp.databricks.com/repos/index.html -33,NS-1,Network Security,All Purpose Cluster Public Keys,-1,High,"Remote SSH access to clusters is discouraged, use web terminal instead",1,1,1,1,0,Check if ssh_public_keys is configured on any cluster,curl --netrc -X GET \ https:///api/2.0/clusters/list \ | jq .,https://docs.databricks.com/clusters/web-terminal.html,https://learn.microsoft.com/en-us/azure/databricks/clusters/web-terminal,https://docs.gcp.databricks.com/clusters/web-terminal.html -34,NS-2,Network Security,Job Cluster Public Keys,-1,High,"Remote SSH access to clusters is discouraged, use web terminal instead",1,1,1,1,0,Check if ssh_public_keys is configured on any job cluster,curl --netrc -X GET \ https:///api/2.0/jobs/list \ | jq,https://docs.databricks.com/clusters/web-terminal.html,https://learn.microsoft.com/en-us/azure/databricks/clusters/web-terminal,https://docs.gcp.databricks.com/clusters/web-terminal.html -35,NS-3,Network Security,Private Link,-1,High,Configure private network connectivity,1,1,1,1,0,Check if private_access_settings_id is set for the workspace,curl -n -X GET 'https://accounts.cloud.databricks.com/api/2.0/accounts//workspaces',https://docs.databricks.com/administration-guide/cloud-configurations/aws/privatelink.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/cloud-configurations/azure/private-link,https://cloud.google.com/vpc/docs/private-access-options -36,NS-4,Network Security,BYOVPC (or BYO Vnet or Vnet Injection),-1,Medium,Deploy with a customer-managed VPC ( BYO Vnet or Vnet Injection)to allow implementation of data exfiltration protections and VPC endpoints,1,1,1,1,0,Check if network_id is set for this workspace,curl -n -X GET 'https://accounts.cloud.databricks.com/api/2.0/accounts//workspaces',https://docs.databricks.com/administration-guide/cloud-configurations/aws/customer-managed-vpc.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/cloud-configurations/azure/vnet-inject,https://docs.gcp.databricks.com/administration-guide/cloud-configurations/gcp/customer-managed-vpc.html -37,NS-5,Network Security,IP Access List,-1,Medium,Configure IP access lists that restrict the IP addresses that can authenticate to Databricks to protect against data exfiltration and account takeover,1,1,1,1,0,Check if ip-access-lists are configured and enabled,curl --netrc -X GET \ https:///api/2.0/ip-access-lists,https://docs.databricks.com/security/network/ip-access-list.html#add-an-ip-access-list,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/access-control/tokens#--set-maximum-lifetime-of-new-tokens-rest-api-only,https://docs.gcp.databricks.com/security/network/ip-access-list.html -38,IA-5,Identity & Access,Maximum lifetime of new tokens,-1,Medium,Configure maximum lifetime for all future tokens,1,1,1,1,0,Check workspace-conf for maxTokenLifetimeDays In Workspace setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=maxTokenLifetimeDays',https://docs.databricks.com/administration-guide/access-control/tokens.html#lifetime,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/access-control/tokens#--set-maximum-lifetime-of-new-tokens-rest-api-only,https://docs.gcp.databricks.com/administration-guide/access-control/tokens.html#lifetime -39,NS-6,Network Security,Secure cluster connectivity,-1,Medium,Configure Secure cluster connectivity (No Public IP / NPIP),0,1,0,1,0,Check if enableNoPublicIp are configured and enabled,curl --netrc -X GET \ https://accounts.azuredatabricks.net/api/2.0/accounts//workspaces,,https://learn.microsoft.com/en-us/azure/databricks/security/secure-cluster-connectivity, -40,GOV-13,Governance,Enforce User Isolation,-1,Medium,Enforce user isolation cluster types on a workspace,1,1,1,1,0,Check workspace-conf for enforceUserIsolation In Workspace setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys= enforceUserIsolation',https://docs.databricks.com/security/enforce-user-isolation.html,https://learn.microsoft.com/en-us/azure/databricks/security/enforce-user-isolation,https://docs.gcp.databricks.com/security/enforce-user-isolation.html -41,IA-6,Identity & Access,Tokens with a lifetime more than the maximum lifetime set,-1,Medium,Update maximum lifetime for all old tokens,1,1,1,1,0,Compare workspace-conf for maxTokenLifetimeDays in workspace setting against all tokens expiration days,curl --netrc -X GET \ https:///api/2.0/token/list | jq .,https://docs.databricks.com/administration-guide/access-control/tokens.html#manage-personal-access-tokens,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/access-control/tokens#manage-personal-access-tokens,https://docs.gcp.databricks.com/administration-guide/access-control/tokens.html#manage-personal-access-tokens -42,IA-7,Identity & Access,Use service principals,1,Medium,Run production workloads with service principals,1,1,1,1,0,Compare workspace-conf for maxTokenLifetimeDays in workspace setting against all tokens expiration days,curl --netrc -X GET \ https:///api/2.0/token/list | jq .,https://docs.databricks.com/administration-guide/users-groups/service-principals.html,https://learn.microsoft.com/en-us/azure/databricks/tutorials/run-jobs-with-service-principalshttps://docs.gcp.databricks.com/administration-guide/users-groups/service-principals.html,https://docs.gcp.databricks.com/administration-guide/users-groups/service-principals.html -43,GOV-14,Governance,Enable Enforce ImdsV2,-1,Low,Enforce AWS Instance Metadata Service v2 on a workspace,1,0,0,1,0,Check workspace-conf for enableEnforceImdsV2 In Workspace setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys= enableEnforceImdsV2',https://docs.databricks.com/administration-guide/cloud-configurations/aws/imdsv2.html,, -44,DP-6,Data Protection,Notebook export ,-1,Low,Disable exporting notebooks and cells within notebooks,1,1,1,1,0,Check workspace-conf for enableExportNotebook setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enableExportNotebook',https://docs.databricks.com/administration-guide/workspace/notebooks.html#manage-download-results,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/workspace/notebooks#manage-download-results,https://docs.gcp.databricks.com/administration-guide/workspace/notebooks.html#manage-download-results -45,DP-7,Data Protection,Notebook Table Clipboard Features,-1,Low,Disable the ability of users to copy tabular data to the clipboard via the Notebooks UI,1,1,1,1,0,Check workspace-conf for enableNotebookTableClipboard setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enableNotebookTableClipboard',https://docs.databricks.com/administration-guide/workspace/notebooks.html#manage-download-results,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/workspace/notebooks#manage-download-results,https://docs.gcp.databricks.com/administration-guide/workspace/notebooks.html#manage-download-results -46,INFO-12,Informational,Manage third-party iFraming prevention,-1,Low,Enable third-party iFraming prevention,1,1,1,1,0,Check workspace-conf for enable-X-Frame-Options In Workspace setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enable-X-Frame-Options',https://docs.databricks.com/administration-guide/workspace/security.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/workspace/security,https://docs.gcp.databricks.com/administration-guide/workspace/security.html -47,INFO-13,Informational,Manage MIME type sniffing prevention,-1,Low,Enable MIME type sniffing prevention,1,1,1,1,0,Check workspace-conf for enable-X-Content-Type-Options In Workspace setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enable-X-Content-Type-Options',https://docs.databricks.com/administration-guide/workspace/security.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/workspace/security,https://docs.gcp.databricks.com/administration-guide/workspace/security.html -48,INFO-14,Informational,Manage XSS attack page rendering prevention,-1,Low,Enable XSS attack page rendering prevention,1,1,1,1,0,Check workspace-conf for enable-X-XSS-Protection In Workspace setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enable-X-XSS-Protection',https://docs.databricks.com/administration-guide/workspace/security.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/workspace/security,https://docs.gcp.databricks.com/administration-guide/workspace/security.html -49,DP-8,Data Protection,Store Interactive Notebook Results in Customer Account,-1,Medium,Enable store interactive notebook results in your account,1,1,1,1,0,Check workspace-conf for storeInteractiveNotebookResultsInCustomerAccount setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=storeInteractiveNotebookResultsInCustomerAccount',https://docs.databricks.com/administration-guide/workspace/notebooks.html#manage-where-notebook-results-are-stored,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/workspace/notebooks#manage-where-notebook-results-are-stored,https://docs.gcp.databricks.com/administration-guide/workspace/notebooks.html#manage-where-notebook-results-are-stored -50,GOV-15,Governance,Enable verbose audit logs,-1,Medium,Enable verbose audit logs,1,1,1,1,0,Check workspace-conf for enableVerboseAuditLogs setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enableVerboseAuditLogs',https://docs.databricks.com/administration-guide/account-settings/audit-logs.html#configure-verbose-audit-logs,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/account-settings/audit-logs#--configure-verbose-audit-logs,https://docs.gcp.databricks.com/administration-guide/account-settings/audit-logs.html#configure-verbose-audit-logs -51,DP-9,Data Protection,FileStore endpoint ,-1,Medium,Review and disable FileStore endpoint in Admin Console Workspace settings,1,1,1,1,0,Check workspace-conf for enableFileStoreEndpoint setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enableFileStoreEndpoint',https://docs.databricks.com/dbfs/filestore.html#filestore,https://learn.microsoft.com/en-us/azure/databricks/dbfs/filestore,https://docs.gcp.databricks.com/dbfs/filestore.html#filestore -52,INFO-15,Informational,Store code in Git,-1,High,Enable git versioning for notebooks,1,1,1,1,0,Check workspace-conf for enableNotebookGitVersioning setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enableNotebookGitVersioning',https://docs.databricks.com/repos/index.html,https://learn.microsoft.com/en-us/azure/databricks/repos/index,https://docs.gcp.databricks.com/repos/index.html \ No newline at end of file +id,check_id,category,check,evaluation_value,severity,recommendation,aws,azure,gcp,enable,alert,logic,api,aws_doc_url,azure_doc_url,gcp_doc_url +1,DP-1,Data Protection,Secrets management,1,Medium,Store and use secrets securely,1,1,1,1,0,Check if there are any secrets configured in the workspace by listing scopes and looking for the secrets count against the configured baseline value in those scopes,curl --netrc -X GET \ https:///api/2.0/secrets/scopes/list,https://docs.databricks.com/security/secrets/index.html,https://learn.microsoft.com/en-us/azure/databricks/security/secrets/,https://docs.gcp.databricks.com/security/secrets/index.html +2,DP-2,Data Protection,Cluster encryption,-1,Low,All clusters should enable local disk encryption,1,1,0,1,0,Check if enable_local_disk_encryption is set as false for any cluster,curl --netrc -X GET \ https:///api/2.0/clusters/list,https://docs.databricks.com/clusters/configure.html#local-disk-encryption,https://learn.microsoft.com/en-us/azure/databricks/clusters/configure#--local-disk-encryption,https://docs.gcp.databricks.com/clusters/configure.html#local-disk-encryption +3,DP-3,Data Protection,BYOK,-1,Low,Add a customer-managed key for managed services and workspace storage,1,0,0,1,0,Check if storage_customer_managed_key_id and managed_services_customer_managed_key_id are set for each workspace,curl -n -X GET 'https://accounts.cloud.databricks.com/api/2.0/accounts//workspaces',https://docs.databricks.com/security/keys/customer-managed-keys-storage-aws.html,https://learn.microsoft.com/en-us/azure/databricks/security/keys/customer-managed-keys,N/A +4,DP-4,Data Protection,Object storage encryption,-1,High,Encrypt object storage data buckets and restrict access,1,1,1,1,0,Manual check (Update cofiguration status using /notebooks/Setup/8. update_workspace_configuration notebook),Manual check,https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucket-encryption.html,https://learn.microsoft.com/en-us/azure/storage/common/storage-service-encryption,https://cloud.google.com/storage/docs/encryption +5,DP-5,Data Protection,Results downloading,-1,High,Disable download button for notebook results,1,1,1,1,0,Check workspace-conf for enableResultsDownloading setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enableResultsDownloading',https://docs.databricks.com/administration-guide/workspace/notebooks.html#manage-download-results,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/workspace/notebooks#manage-download-results,https://docs.gcp.databricks.com/administration-guide/workspace/notebooks.html#manage-download-results +6,GOV-1,Governance,Cluster policies,-1,High,Configure cluster policies to enforce data access patterns and control costs,1,1,1,1,0,Check if policy_id is set for clusters,curl --netrc -X GET \ https:///api/2.0/clusters/list \ | jq .,https://docs.databricks.com/administration-guide/clusters/policies.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/clusters/policies,https://docs.gcp.databricks.com/administration-guide/clusters/policies.html +7,GOV-2,Governance,PAT Tokens about to expire,7,High,"Set lifetime limit, but also regularly review PAT tokens to avoid expired tokens impacting authentications",1,1,1,1,0,Check each token expiry_time and report if the expiry_time is within configured days,curl --netrc -X GET \ https:///api/2.0/token/list | jq .,https://docs.databricks.com/administration-guide/access-control/tokens.html#manage-personal-access-tokens,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/access-control/tokens#manage-personal-access-tokens,https://docs.gcp.databricks.com/administration-guide/access-control/tokens.html#manage-personal-access-tokens +8,GOV-3,Governance,Log delivery configurations,-1,High,Configure Databricks audit log delivery,1,1,1,1,0,"Check account log-delivery configuration and look for audit log config with log_type set as ""AUDIT_LOGS"" and status set as ""ENABLED""",curl -netrc -X GET \ 'https://accounts.cloud.databricks.com/api/2.0/accounts//log-delivery',https://docs.databricks.com/administration-guide/account-settings/audit-logs.html#configure-audit-log-delivery,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/account-settings/audit-logs#configure-audit-log-delivery,https://docs.gcp.databricks.com/administration-guide/account-settings/audit-logs.html#configure-audit-log-delivery +9,GOV-4,Governance,Long running clusters,24,Medium,Automatically restart long running clusters,1,1,1,1,0,Check each running cluster's last restart time till now and report on clusters that were running longer than the configured number of days without a restart,curl --netrc -X GET \ https:///api/2.0/clusters/list \ | jq .,https://docs.databricks.com/clusters/clusters-manage.html#restart-a-cluster-to-update-it-with-the-latest-images,https://learn.microsoft.com/en-us/azure/databricks/clusters/clusters-manage#--restart-a-cluster-to-update-it-with-the-latest-images,https://docs.gcp.databricks.com/clusters/clusters-manage.html#restart-a-cluster-to-update-it-with-the-latest-images +10,GOV-5,Governance,Deprecated runtime versions,-1,High,Deprecated runtime version detected. Please update your cluster runtimes to Databricks supported runtimes,1,1,1,1,0,List clusters with spark version that is not in the supported spark versions,curl --netrc -X GET \ https:///api/2.0/clusters/spark-versions,https://docs.databricks.com/release-notes/runtime/releases.html,https://learn.microsoft.com/en-us/azure/databricks/release-notes/runtime/releases,https://docs.gcp.databricks.com/release-notes/runtime/releases.html +11,GOV-6,Governance,All Purpose Cluster custom tags,-1,Low,Configure cluster tagging to monitor usage and enable charge-back,1,1,1,1,0,check if custom_tags is set for clusters,curl --netrc -X GET \ https:///api/2.0/clusters/list \ | jq .,https://docs.databricks.com/administration-guide/account-settings/usage-detail-tags-aws.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/account-settings/usage-detail-tags-azure,https://docs.gcp.databricks.com/administration-guide/account-settings-gcp/usage-detail-tags-gcp.html +12,GOV-7,Governance,Job Cluster custom tags,-1,Low,Configure job tagging to monitor usage and enable charge-back,1,1,1,1,0,Check if settings.new_cluster.custom_tags is not null for job clusters,curl --netrc -X GET \ https:///api/2.0/jobs/list \ | jq,https://docs.databricks.com/administration-guide/account-settings/usage-detail-tags-aws.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/account-settings/usage-detail-tags-azure,https://docs.gcp.databricks.com/administration-guide/account-settings-gcp/usage-detail-tags-gcp.html +13,GOV-8,Governance,All Purpose Cluster log configuration,-1,Low,Configure Databricks cluster log delivery,1,1,1,1,0,Check if cluster_log_conf is set for clusters,curl --netrc -X GET \ https:///api/2.0/clusters/list \ | jq .,https://docs.databricks.com/clusters/configure.html#cluster-log-delivery-1,https://learn.microsoft.com/en-us/azure/databricks/clusters/configure#cluster-log-delivery,https://docs.gcp.databricks.com/clusters/configure.html#cluster-log-delivery-1 +14,GOV-9,Governance,Job Cluster log configuration,-1,Low,Configure Databricks job custer log delivery,1,1,1,1,0,Check if cluster_log_conf is set for job clusters,curl --netrc -X GET \ https:///api/2.0/jobs/list \ | jq,https://docs.databricks.com/clusters/configure.html#cluster-log-delivery-1,https://learn.microsoft.com/en-us/azure/databricks/clusters/configure#cluster-log-delivery,https://docs.gcp.databricks.com/clusters/configure.html#cluster-log-delivery-1 +15,GOV-10,Governance,Managed Tables,1,Low,The DBFS root is not intended for production customer data,1,1,1,1,0,Check the /user/hive/warehouse/ folder for any data folders stored more than the configured value,"curl --netrc -X GET \ https:///api/2.0/dbfs/list \ --data '{ ""path"": ""/user/hive/warehouse/"" }'",https://docs.databricks.com/data/databricks-file-system.html#configuration-and-usage-recommendations,https://learn.microsoft.com/en-us/azure/databricks/data/databricks-file-system#configuration-and-usage-recommendations,https://docs.gcp.databricks.com/dbfs/index.html#configuration-and-usage-recommendations +16,GOV-11,Governance,Mounts,1,Low,Avoid FUSE mounts for accessing production data,1,1,1,1,0,Check for mnt paths in dbutils.fs.mounts() and report if there are datasources loaded as FUSE mounts to the workspace than the configured value,dbutils.fs.mounts(),https://docs.databricks.com/data/databricks-file-system.html#configuration-and-usage-recommendations,https://learn.microsoft.com/en-us/azure/databricks/data/databricks-file-system#configuration-and-usage-recommendations,https://docs.gcp.databricks.com/dbfs/index.html#configuration-and-usage-recommendations +17,GOV-12,Governance,UC enabled clusters,-1,High,Use UC enabled clusters,1,1,0,1,0,Check if there are clusters without data_security_mode as (USER_ISOLATION or SINGLE_USER) or data_security_mode as none,curl --netrc -X GET \ https:///api/2.0/clusters/list \ | jq .,https://docs.databricks.com/data-governance/unity-catalog/index.html#cluster-access-modes-for-unity-catalog,https://learn.microsoft.com/en-us/azure/databricks/data-governance/unity-catalog/#cluster-access-modes-for-unity-catalog,https://docs.gcp.databricks.com/data-governance/unity-catalog/index.html#cluster-access-modes-for-unity-catalog +18,IA-1,Identity & Access,SSO,-1,High,Authenticate via single sign-on and leverage multi-factor authentication,1,1,1,1,0,Manual check (Update cofiguration status using /notebooks/Setup/8. update_workspace_configuration notebook),Manual check,https://docs.databricks.com/administration-guide/users-groups/single-sign-on/index.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/users-groups/single-sign-on/,https://docs.gcp.databricks.com/administration-guide/users-groups/single-sign-on/index.html +19,IA-2,Identity & Access,SCIM,-1,High,Keep an up-to-date user list by using SCIM,1,1,1,1,0,Manual check (Update cofiguration status using /notebooks/Setup/8. update_workspace_configuration notebook),Manual check,https://docs.databricks.com/administration-guide/users-groups/scim/index.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/users-groups/scim/index,https://docs.gcp.databricks.com/administration-guide/users-groups/scim/index.html +20,IA-3,Identity & Access,Table Access Control,-1,High,Enable Table Access Control in admin settings so that you can utilize Table ACL clusters that enforce user isolation,1,1,1,1,0,Manual check (Update cofiguration status using /notebooks/Setup/8. update_workspace_configuration notebook),Manual check,https://docs.databricks.com/administration-guide/access-control/table-acl.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/access-control/table-acl,https://docs.gcp.databricks.com/administration-guide/access-control/table-acl.html +21,IA-4,Identity & Access,PAT Tokens with no lifetime limit,-1,Medium,Configure maximum token lifetimes for future tokens using token management,1,1,1,1,0,Check each token expiry_time and report if the expiry_time is not within the configured number of days or set to never expire by using -1 as the value,curl --netrc -X GET \ https:///api/2.0/token/list,https://docs.databricks.com/administration-guide/access-control/tokens.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/access-control/tokens#manage-personal-access-tokens,https://docs.gcp.databricks.com/administration-guide/access-control/tokens.html +22,INFO-1,Informational,Instance Pool custom tag,-1,Low,Configure tagging to monitor usage and enable charge-back,1,1,1,1,0,Check if instance-pools have custom tags configured,curl --netrc -X GET \ https:///api/2.0/instance-pools/list | jq .,https://docs.databricks.com/clusters/instance-pools/configure.html#pool-tags,https://learn.microsoft.com/en-us/azure/databricks/clusters/instance-pools/configure#pool-tags,https://docs.gcp.databricks.com/clusters/instance-pools/configure.html#pool-tags +23,INFO-2,Informational,Max concurrent runs,5,Low,Limit the number of parallel runs for a given job to avoid resource contention,1,1,1,1,0,Check if max_concurrent_runs configuration for each job is less than configured value,curl --netrc -X GET \ https:///api/2.0/jobs/list \ | jq,https://docs.databricks.com/data-engineering/jobs/jobs.html#max-concurrent-runs,https://learn.microsoft.com/en-us/azure/databricks/data-engineering/jobs/jobs#max-concurrent-runs,https://docs.gcp.databricks.com/data-engineering/jobs/jobs.html#max-concurrent-runs +24,INFO-3,Informational,Global libraries,-1,Low,"Global libraries are discouraged for security reasons. Use cluster libraries or notebook-scoped libraries to improve startup time for clusters that don't require that library, and to improve flexibility where that library is not required.",1,1,1,1,0,Check if is_library_for_all_clusters is set as true for any library that is configured for clusters,curl --netrc -X GET \ https:///api/2.0/libraries/all-cluster-statuses | jq .,https://docs.databricks.com/libraries/cluster-libraries.html,https://learn.microsoft.com/en-us/azure/databricks/libraries/cluster-libraries,https://docs.gcp.databricks.com/libraries/cluster-libraries.html +25,INFO-4,Informational,User Privileges,5,Low,Limit number of users with cluster create privileges,1,1,1,1,0,"Check entitlements.value for each user and look if they have 'allow-cluster-create', 'allow-instance-pool-create' as permission, and count the number of users with such permissions to report if that crossed the configured threshold.",curl --netrc -X GET \ https:///api/2.0/preview/scim/v2/Users,https://docs.databricks.com/security/access-control/cluster-acl.html,https://learn.microsoft.com/en-us/azure/databricks/security/access-control/cluster-acl,https://docs.gcp.databricks.com/security/access-control/cluster-acl.html +26,INFO-5,Informational,Global Init Script,-1,Medium,"Global init scripts are discouraged for security, improved startup and flexibility reasons",1,1,1,1,0,Check if there are any global-init-scripts configured,curl --netrc -X GET \ https:///api/2.0/global-init-scripts,https://docs.databricks.com/clusters/init-scripts.html#init-script-types,https://learn.microsoft.com/en-us/azure/databricks/clusters/init-scripts#init-script-types,https://docs.gcp.databricks.com/clusters/init-scripts.html#global-init-scripts +27,INFO-6,Informational,Admin Count,2,Low,Limit the number of admin accounts so that most users are not admins,1,1,1,1,0,"Check members count of diplayName ""admins"" in groups to report if the count is more than the configured threshold",curl --netrc -X GET \ https:///api/2.0/preview/scim/v2/Groups,https://docs.databricks.com/administration-guide/users-groups/users.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/users-groups/users,https://docs.gcp.databricks.com/administration-guide/users-groups/users.html +28,INFO-7,Informational,VPC (or VNET) Peering,-1,Medium,VPC (or VNET) endpoints or Private Link are recommended,1,1,1,1,0,Manual check (Update cofiguration status using /notebooks/Setup/8. update_workspace_configuration notebook),Manual check,https://docs.aws.amazon.com/vpc/latest/privatelink/create-interface-endpoint.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/cloud-configurations/azure/vnet-inject,https://cloud.google.com/vpc/docs/private-access-options +29,INFO-8,Informational,Job View Acls,-1,High,Use Job Visibility Control to prevent users from viewing jobs where they do not have permissions,1,1,1,1,0,Check workspace-conf for enableJobViewAcls setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enableJobViewAcls',https://docs.databricks.com/administration-guide/access-control/jobs-acl.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/access-control/jobs-acl,https://docs.gcp.databricks.com/administration-guide/access-control/jobs-acl.html +30,INFO-9,Informational,Cluster View Acls,-1,High,Use Cluster Visibility Control to prevent users from viewing clusters where they do not have permissions,1,1,1,1,0,Check workspace-conf for enforceClusterViewAcls setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enforceClusterViewAcls',https://docs.databricks.com/security/access-control/cluster-acl.html,https://learn.microsoft.com/en-us/azure/databricks/security/access-control/cluster-acl,https://docs.gcp.databricks.com/security/access-control/cluster-acl.html +31,INFO-10,Informational,Workspace View Acls,-1,High,Use Workspace Visibility Control to prevent users from viewing notebooks and other objects where they do not have permissions in the workspace,1,1,1,1,0,Check workspace-conf for enforceWorkspaceViewAcls setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enforceWorkspaceViewAcls',https://docs.databricks.com/security/access-control/workspace-acl.html,https://learn.microsoft.com/en-us/azure/databricks/security/access-control/workspace-acl,https://docs.gcp.databricks.com/security/access-control/workspace-acl.html +32,INFO-11,Informational,Project Type In Workspace,-1,High,It is recommended to store code in Git repos,1,1,1,1,0,Check workspace-conf for enableProjectTypeInWorkspace setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enableProjectTypeInWorkspace',https://docs.databricks.com/repos/index.html,https://learn.microsoft.com/en-us/azure/databricks/repos/index,https://docs.gcp.databricks.com/repos/index.html +33,NS-1,Network Security,All Purpose Cluster Public Keys,-1,High,"Remote SSH access to clusters is discouraged, use web terminal instead",1,1,1,1,0,Check if ssh_public_keys is configured on any cluster,curl --netrc -X GET \ https:///api/2.0/clusters/list \ | jq .,https://docs.databricks.com/clusters/web-terminal.html,https://learn.microsoft.com/en-us/azure/databricks/clusters/web-terminal,https://docs.gcp.databricks.com/clusters/web-terminal.html +34,NS-2,Network Security,Job Cluster Public Keys,-1,High,"Remote SSH access to clusters is discouraged, use web terminal instead",1,1,1,1,0,Check if ssh_public_keys is configured on any job cluster,curl --netrc -X GET \ https:///api/2.0/jobs/list \ | jq,https://docs.databricks.com/clusters/web-terminal.html,https://learn.microsoft.com/en-us/azure/databricks/clusters/web-terminal,https://docs.gcp.databricks.com/clusters/web-terminal.html +35,NS-3,Network Security,Private Link,-1,High,Configure private network connectivity,1,1,1,1,0,Check if private_access_settings_id is set for the workspace,curl -n -X GET 'https://accounts.cloud.databricks.com/api/2.0/accounts//workspaces',https://docs.databricks.com/administration-guide/cloud-configurations/aws/privatelink.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/cloud-configurations/azure/private-link,https://cloud.google.com/vpc/docs/private-access-options +36,NS-4,Network Security,BYOVPC (or BYO Vnet or Vnet Injection),-1,Medium,Deploy with a customer-managed VPC ( BYO Vnet or Vnet Injection)to allow implementation of data exfiltration protections and VPC endpoints,1,1,1,1,0,Check if network_id is set for this workspace,curl -n -X GET 'https://accounts.cloud.databricks.com/api/2.0/accounts//workspaces',https://docs.databricks.com/administration-guide/cloud-configurations/aws/customer-managed-vpc.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/cloud-configurations/azure/vnet-inject,https://docs.gcp.databricks.com/administration-guide/cloud-configurations/gcp/customer-managed-vpc.html +37,NS-5,Network Security,IP Access List,-1,Medium,Configure IP access lists that restrict the IP addresses that can authenticate to Databricks to protect against data exfiltration and account takeover,1,1,1,1,0,Check if ip-access-lists are configured and enabled,curl --netrc -X GET \ https:///api/2.0/ip-access-lists,https://docs.databricks.com/security/network/ip-access-list.html#add-an-ip-access-list,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/access-control/tokens#--set-maximum-lifetime-of-new-tokens-rest-api-only,https://docs.gcp.databricks.com/security/network/ip-access-list.html +38,IA-5,Identity & Access,Maximum lifetime of new tokens,-1,Medium,Configure maximum lifetime for all future tokens,1,1,1,1,0,Check workspace-conf for maxTokenLifetimeDays In Workspace setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=maxTokenLifetimeDays',https://docs.databricks.com/administration-guide/access-control/tokens.html#lifetime,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/access-control/tokens#--set-maximum-lifetime-of-new-tokens-rest-api-only,https://docs.gcp.databricks.com/administration-guide/access-control/tokens.html#lifetime +39,NS-6,Network Security,Secure cluster connectivity,-1,Medium,Configure Secure cluster connectivity (No Public IP / NPIP),0,1,0,1,0,Check if enableNoPublicIp are configured and enabled,curl --netrc -X GET \ https://accounts.azuredatabricks.net/api/2.0/accounts//workspaces,N/A,https://learn.microsoft.com/en-us/azure/databricks/security/secure-cluster-connectivity, +40,GOV-13,Governance,Enforce User Isolation,-1,Medium,Enforce user isolation cluster types on a workspace,1,1,1,1,0,Check workspace-conf for enforceUserIsolation In Workspace setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys= enforceUserIsolation',https://docs.databricks.com/security/enforce-user-isolation.html,https://learn.microsoft.com/en-us/azure/databricks/security/enforce-user-isolation,https://docs.gcp.databricks.com/security/enforce-user-isolation.html +41,IA-6,Identity & Access,Tokens with a lifetime more than the maximum lifetime set,-1,Medium,Update maximum lifetime for all old tokens,1,1,1,1,0,Compare workspace-conf for maxTokenLifetimeDays in workspace setting against all tokens expiration days,curl --netrc -X GET \ https:///api/2.0/token/list | jq .,https://docs.databricks.com/administration-guide/access-control/tokens.html#manage-personal-access-tokens,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/access-control/tokens#manage-personal-access-tokens,https://docs.gcp.databricks.com/administration-guide/access-control/tokens.html#manage-personal-access-tokens +42,IA-7,Identity & Access,Use service principals,1,Medium,Run production workloads with service principals,1,1,1,1,0,Compare workspace-conf for maxTokenLifetimeDays in workspace setting against all tokens expiration days,curl --netrc -X GET \ https:///api/2.0/token/list | jq .,https://docs.databricks.com/administration-guide/users-groups/service-principals.html,https://learn.microsoft.com/en-us/azure/databricks/tutorials/run-jobs-with-service-principalshttps://docs.gcp.databricks.com/administration-guide/users-groups/service-principals.html,https://docs.gcp.databricks.com/administration-guide/users-groups/service-principals.html +43,GOV-14,Governance,Enable Enforce ImdsV2,-1,Low,Enforce AWS Instance Metadata Service v2 on a workspace,1,0,0,1,0,Check workspace-conf for enableEnforceImdsV2 In Workspace setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys= enableEnforceImdsV2',https://docs.databricks.com/administration-guide/cloud-configurations/aws/imdsv2.html,, +44,DP-6,Data Protection,Notebook export,-1,Low,Disable exporting notebooks and cells within notebooks,1,1,1,1,0,Check workspace-conf for enableExportNotebook setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enableExportNotebook',https://docs.databricks.com/administration-guide/workspace/notebooks.html#manage-download-results,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/workspace/notebooks#manage-download-results,https://docs.gcp.databricks.com/administration-guide/workspace/notebooks.html#manage-download-results +45,DP-7,Data Protection,Notebook Table Clipboard Features,-1,Low,Disable the ability of users to copy tabular data to the clipboard via the Notebooks UI,1,1,1,1,0,Check workspace-conf for enableNotebookTableClipboard setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enableNotebookTableClipboard',https://docs.databricks.com/administration-guide/workspace/notebooks.html#manage-download-results,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/workspace/notebooks#manage-download-results,https://docs.gcp.databricks.com/administration-guide/workspace/notebooks.html#manage-download-results +46,INFO-12,Informational,Manage third-party iFraming prevention,-1,Low,Enable third-party iFraming prevention,1,1,1,1,0,Check workspace-conf for enable-X-Frame-Options In Workspace setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enable-X-Frame-Options',https://docs.databricks.com/administration-guide/workspace/security.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/workspace/security,https://docs.gcp.databricks.com/administration-guide/workspace/security.html +47,INFO-13,Informational,Manage MIME type sniffing prevention,-1,Low,Enable MIME type sniffing prevention,1,1,1,1,0,Check workspace-conf for enable-X-Content-Type-Options In Workspace setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enable-X-Content-Type-Options',https://docs.databricks.com/administration-guide/workspace/security.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/workspace/security,https://docs.gcp.databricks.com/administration-guide/workspace/security.html +48,INFO-14,Informational,Manage XSS attack page rendering prevention,-1,Low,Enable XSS attack page rendering prevention,1,1,1,1,0,Check workspace-conf for enable-X-XSS-Protection In Workspace setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enable-X-XSS-Protection',https://docs.databricks.com/administration-guide/workspace/security.html,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/workspace/security,https://docs.gcp.databricks.com/administration-guide/workspace/security.html +49,DP-8,Data Protection,Store Interactive Notebook Results in Customer Account,-1,Medium,Enable store interactive notebook results in your account,1,1,1,1,0,Check workspace-conf for storeInteractiveNotebookResultsInCustomerAccount setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=storeInteractiveNotebookResultsInCustomerAccount',https://docs.databricks.com/administration-guide/workspace/notebooks.html#manage-where-notebook-results-are-stored,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/workspace/notebooks#manage-where-notebook-results-are-stored,https://docs.gcp.databricks.com/administration-guide/workspace/notebooks.html#manage-where-notebook-results-are-stored +50,GOV-15,Governance,Enable verbose audit logs,-1,Medium,Enable verbose audit logs,1,1,1,1,0,Check workspace-conf for enableVerboseAuditLogs setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enableVerboseAuditLogs',https://docs.databricks.com/administration-guide/account-settings/audit-logs.html#configure-verbose-audit-logs,https://learn.microsoft.com/en-us/azure/databricks/administration-guide/account-settings/audit-logs#--configure-verbose-audit-logs,https://docs.gcp.databricks.com/administration-guide/account-settings/audit-logs.html#configure-verbose-audit-logs +51,DP-9,Data Protection,FileStore endpoint,-1,Medium,Review and disable FileStore endpoint in Admin Console Workspace settings,1,1,1,1,0,Check workspace-conf for enableFileStoreEndpoint setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enableFileStoreEndpoint',https://docs.databricks.com/dbfs/filestore.html#filestore,https://learn.microsoft.com/en-us/azure/databricks/dbfs/filestore,https://docs.gcp.databricks.com/dbfs/filestore.html#filestore +52,INFO-15,Informational,Store code in Git,-1,High,Enable git versioning for notebooks,1,1,1,1,0,Check workspace-conf for enableNotebookGitVersioning setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys=enableNotebookGitVersioning',https://docs.databricks.com/repos/index.html,https://learn.microsoft.com/en-us/azure/databricks/repos/index,https://docs.gcp.databricks.com/repos/index.html +53,GOV-16,Governance,Workspace Unity Catalog metastore assignment,-1,Medium,Enable a workspace for Unity Catalog by assigning a Unity Catalog metastore,1,1,1,1,0,Check if current-metastore-assignment has the workspace assigned to metastore_id,curl --netrc -X GET \ https:///api/2.1/unity-catalog/current-metastore-assignment,https://docs.databricks.com/data-governance/unity-catalog/enable-workspaces.html,https://learn.microsoft.com/en-us/azure/databricks/data-governance/unity-catalog/enable-workspaces,https://docs.gcp.databricks.com/data-governance/unity-catalog/enable-workspaces.html +54,GOV-17,Governance,Lifetime of metastore delta sharing recipient token,-1,High,Set the lifetime of delta sharing recipient tokens,1,1,1,1,0,Check if delta_sharing_recipient_token_lifetime_in_seconds is set less than 90 days where delta_sharing_scope is INTERNAL_AND_EXTERNAL ,curl --netrc -X GET \ https:///api/2.1/unity-catalog/metastore_summary,https://docs.databricks.com/data-sharing/create-recipient.html#modify-the-recipient-token-lifetime,https://learn.microsoft.com/en-us/azure/databricks/data-sharing/create-recipient#modify-recipient-token-lifetime,https://docs.gcp.databricks.com/data-sharing/create-recipient.html#modify-the-recipient-token-lifetime +55,GOV-18,Governance,Delta sharing IP Access Lists,-1,Medium,Configure Delta Sharing IP access lists to restrict recipient access to trusted IP addresses,1,1,1,1,0,"Check if ip_access_list is present on share recipients for authentication_type ""TOKEN""",curl --netrc -X GET \ https:///api/2.1/unity-catalog/recipients,https://docs.databricks.com/data-sharing/access-list.html#use-ip-access-lists-to-restrict-delta-sharing-recipient-access-open-sharing,https://learn.microsoft.com/en-gb/azure/databricks/data-sharing/access-list,https://docs.gcp.databricks.com/data-sharing/create-recipient.html#security-considerations-for-tokens +56,GOV-19,Governance,Delta sharing Token Expiration,-1,Medium,Establish a process for rotating credentials Delta sharing Token,1,1,1,1,0,"Check if expiration_time on share recipients for tokens for share with authentication_type ""TOKEN""",curl --netrc -X GET \ https:///api/2.1/unity-catalog/recipients,https://docs.databricks.com/data-sharing/create-recipient.html#security-considerations-for-tokens,https://learn.microsoft.com/en-us/azure/databricks/data-sharing/create-recipient#--security-considerations-for-tokens,https://docs.gcp.databricks.com/data-sharing/access-list.html#security-considerations-for-tokens +57,GOV-20,Governance,Use of Metastore,-1,Low,Create a Unity Catalog metastore,1,1,1,1,0,Check if securable_type = 'METASTORE' exists in metasores,curl --netrc -X GET \ https:///api/2.1/unity-catalog/metastores,https://docs.databricks.com/data-governance/unity-catalog/create-metastore.html,https://learn.microsoft.com/en-us/azure/databricks/data-governance/unity-catalog/create-metastore,https://docs.gcp.databricks.com/data-governance/unity-catalog/create-metastore.html +58,GOV-21,Governance,Metastore Admin,-1,High,Delegate metastore admin to a group who is not the account admin,1,1,1,1,0,Check if securable_type = 'METASTORE' and owner != created_by for a metastore,curl --netrc -X GET \ https:///api/2.1/unity-catalog/metastores,https://docs.databricks.com/data-governance/unity-catalog/manage-privileges/index.html#assign-a-metastore-admin,https://learn.microsoft.com/en-us/azure/databricks/data-governance/unity-catalog/manage-privileges/#--assign-a-metastore-admin,https://docs.gcp.databricks.com/data-governance/unity-catalog/manage-privileges/index.html#assign-a-metastore-admin +59,GOV-22,Informational,Metastore Storage Credentials,-1,Medium,Use external locations rather than using storage credentials directly,1,1,1,0,0,Check if securable_type = 'STORAGE_CREDENTIAL' for storage-credentials ,curl --netrc -X GET \ https:///api/2.1/unity-catalog/storage-credentials,https://docs.databricks.com/data-governance/unity-catalog/manage-external-locations-and-credentials.html,https://learn.microsoft.com/en-us/azure/databricks/data-governance/unity-catalog/manage-external-locations-and-credentials,https://docs.gcp.databricks.com/data-governance/unity-catalog/manage-external-locations-and-credentials.html +60,GOV-23,Governance,UC enabled Data warehouses,-1,Low,Use UC enabled Data warehouses,1,1,1,1,0,Check if disable_uc = true on warehouses,curl --netrc -X GET \ https:///api/2.0/sql/warehouses,https://docs.databricks.com/sql/admin/create-sql-warehouse.html#advanced-options,https://learn.microsoft.com/en-us/azure/databricks/sql/admin/create-sql-warehouse#--advanced-options,https://docs.gcp.databricks.com/sql/admin/create-sql-warehouse.html#advanced-options +61,INFO-17,Informational,Serverless Compute,-1,Low,Enable Serverless Compute,1,1,0,0,0,Check if enable_serverless_compute != true on warehouses ,curl --netrc -X GET \ https:///api/2.0/sql/config/warehouses,https://docs.databricks.com/sql/admin/serverless.html,https://learn.microsoft.com/en-us/azure/databricks/sql/admin/serverless,N/A +62,INFO-18,Informational,Delta Sharing CREATE_RECIPIENT and CREATE_SHARE permissions,-1,Low,Limit who has CREATE_RECIPIENT and CREATE_SHARE permissions on metastore,1,1,1,1,0,Check who has CREATE_RECIPIENT and CREATE_SHARE permission on the metastore,curl --netrc -X GET \ https:///api/2.1/unity-catalog/permissions/{securable_type}/{full_name},https://docs.databricks.com/data-sharing/create-recipient.html,https://learn.microsoft.com/en-us/azure/databricks/data-sharing/create-recipient,https://docs.gcp.databricks.com/data-sharing/create-recipient.html \ No newline at end of file diff --git a/notebooks/Includes/install_sat_sdk.py b/notebooks/Includes/install_sat_sdk.py index da794dfc..c372e43e 100644 --- a/notebooks/Includes/install_sat_sdk.py +++ b/notebooks/Includes/install_sat_sdk.py @@ -1,5 +1,5 @@ # Databricks notebook source -SDK_VERSION='0.1.23' +SDK_VERSION='0.1.25' # COMMAND ---------- diff --git a/notebooks/Includes/workspace_analysis.py b/notebooks/Includes/workspace_analysis.py index aea2bd3f..21cb336f 100644 --- a/notebooks/Includes/workspace_analysis.py +++ b/notebooks/Includes/workspace_analysis.py @@ -903,6 +903,7 @@ def log_check(df): df = df.rdd.map(lambda x: ( re.sub('[\"\'\\\\]', '_',x[0]), x[1])).toDF(['config_name', 'config_id']) logc = df.collect() logc_dict = {'audit_logs' : [[i.config_name, i.config_id] for i in logc]} + print(logc_dict) return (check_id, 0, logc_dict) else: @@ -995,6 +996,224 @@ def uc_check(df): # COMMAND ---------- +check_id='53' # GOV-16 Workspace Unity Catalog metastore assignment +enabled, sbp_rec = getSecurityBestPracticeRecord(check_id, cloud_type) + +def uc_metasore_assignment(df): + if df is not None and not df.rdd.isEmpty(): + uc_metasore = df.collect() + uc_metasore_dict = {i.metastore_id : [i.workspace_id] for i in uc_metasore} + return (check_id, 0, uc_metasore_dict ) + else: + return (check_id, 1, {}) +if enabled: + tbl_name = 'global_temp.unitycatalogmsv2' + '_' + workspace_id + sql=f''' + SELECT metastore_id,workspace_id + FROM {tbl_name} + WHERE workspace_id="{workspaceId}" + + ''' + sqlctrl(workspace_id, sql, uc_metasore_assignment) + +# COMMAND ---------- + +check_id='54' # GOV-17 Lifetime of metastore delta sharing recipient token set less than 90 days +enabled, sbp_rec = getSecurityBestPracticeRecord(check_id, cloud_type) + +def uc_metasore_token(df): + if df is not None and not df.rdd.isEmpty(): + uc_metasore = df.collect() + uc_metasore_dict = {num: [row.name,row.delta_sharing_recipient_token_lifetime_in_seconds] for num,row in enumerate(uc_metasore)} + return (check_id, 1, uc_metasore_dict ) + else: + return (check_id, 0, {}) +if enabled: + tbl_name = 'global_temp.unitycatalogmsv1' + '_' + workspace_id + sql=f''' + SELECT name, delta_sharing_recipient_token_lifetime_in_seconds + FROM {tbl_name} + WHERE delta_sharing_scope ="INTERNAL_AND_EXTERNAL" AND delta_sharing_recipient_token_lifetime_in_seconds < 7776000 + ''' + sqlctrl(workspace_id, sql, uc_metasore_token) + + +# COMMAND ---------- + +check_id='55' # GOV-18 Check if there are any token based sharing without IP access lists ip_access_list +enabled, sbp_rec = getSecurityBestPracticeRecord(check_id, cloud_type) + +def uc_delta_share_ip_accesslist(df): + if df is not None and not df.rdd.isEmpty(): + uc_metasore = df.collect() + uc_metasore_dict = {num: [row.name,row.owner] for num,row in enumerate(uc_metasore)} + return (check_id, 1, uc_metasore_dict ) + else: + return (check_id, 0, {}) +if enabled: + tbl_name = 'global_temp.unitycatalogsharerecipients' + '_' + workspace_id + sql=f''' + SELECT name, owner + FROM {tbl_name} + where authentication_type = 'TOKEN' and ip_access_list is NULL + ''' + sqlctrl(workspace_id, sql, uc_delta_share_ip_accesslist) + + +# COMMAND ---------- + +check_id='56' # GOV-19 Check if Delta sharing Token Expiration +enabled, sbp_rec = getSecurityBestPracticeRecord(check_id, cloud_type) + +def uc_delta_share_expiration_time(df): + if df is not None and not df.rdd.isEmpty(): + uc_metasore = df.collect() + uc_metasore_dict = {num: [row.name,row.owner] for num,row in enumerate(uc_metasore)} + return (check_id, 1, uc_metasore_dict ) + else: + return (check_id, 0, {}) +if enabled: + tbl_name = 'global_temp.unitycatalogsharerecipients' + '_' + workspace_id + sql=f''' + SELECT tokens.* FROM (select explode(tokens) as tokens, full_name, owner + FROM {tbl_name} + WHERE authentication_type = 'TOKEN') WHERE tokens.expiration_time is NULL + ''' + sqlctrl(workspace_id, sql, uc_delta_share_expiration_time) + + +# COMMAND ---------- + +check_id='57' # GOV-20 Check Use of Metastore +enabled, sbp_rec = getSecurityBestPracticeRecord(check_id, cloud_type) + +def uc_metastore(df): + if df is not None and not df.rdd.isEmpty(): + uc_metasore = df.collect() + uc_metasore_dict = {i.name : [i.owner] for i in uc_metasore} + return (check_id, 0, uc_metasore_dict ) + else: + return (check_id, 1, {}) +if enabled: + tbl_name = 'global_temp.unitycatalogmsv1' + '_' + workspace_id + sql=f''' + SELECT name,owner + FROM {tbl_name} + WHERE securable_type = 'METASTORE' + ''' + sqlctrl(workspace_id, sql, uc_metastore) + + +# COMMAND ---------- + +check_id='58' # GOV-21 Check Metastore Admin is also the creator +enabled, sbp_rec = getSecurityBestPracticeRecord(check_id, cloud_type) + +def uc_metastore_owner(df): + if df is not None and not df.rdd.isEmpty(): + uc_metasore = df.collect() + uc_metasore_dict = {i.name : [i.owner, i.created_by] for i in uc_metasore} + return (check_id, 1, uc_metasore_dict ) + else: + return (check_id, 0, {}) +if enabled: + tbl_name = 'global_temp.unitycatalogmsv1' + '_' + workspace_id + sql=f''' + SELECT name,owner,created_by + FROM {tbl_name} + WHERE securable_type = 'METASTORE' and owner == created_by + ''' + sqlctrl(workspace_id, sql, uc_metastore_owner) + + +# COMMAND ---------- + +check_id='59' # GOV-22 Check Metastore Storage Credentials +enabled, sbp_rec = getSecurityBestPracticeRecord(check_id, cloud_type) + +def uc_metastore_storage_creds(df): + if df is not None and not df.rdd.isEmpty(): + uc_metasore = df.collect() + uc_metasore_dict = {num: [row.name,row.owner, row.created_by] for num,row in enumerate(uc_metasore)} + return (check_id, 1, uc_metasore_dict ) + else: + return (check_id, 0, {}) +if enabled: + tbl_name = 'global_temp.unitycatalogcredentials' + '_' + workspace_id + sql=f''' + SELECT name,owner,created_by + FROM {tbl_name} + WHERE securable_type = "STORAGE_CREDENTIAL" + ''' + sqlctrl(workspace_id, sql, uc_metastore_storage_creds) + + +# COMMAND ---------- + +check_id='60' # GOV-23 Check UC enabled Data warehouses +enabled, sbp_rec = getSecurityBestPracticeRecord(check_id, cloud_type) + +def uc_dws(df): + if df is not None and not df.rdd.isEmpty(): + uc_metasore = df.collect() + uc_metasore_dict = {i.name : [i.creator_name] for i in uc_metasore} + + return (check_id, 1, uc_metasore_dict ) + else: + return (check_id, 0, {}) +if enabled: + tbl_name = 'global_temp.dbsql_warehouselistv2' + '_' + workspace_id + sql=f''' + SELECT warehouse.name as name , warehouse.creator_name as creator_name from (select explode(warehouses) as warehouse + FROM {tbl_name} ) + where warehouse.disable_uc = true + ''' + sqlctrl(workspace_id, sql, uc_dws) + + +# COMMAND ---------- + +check_id='61' # INFO-17 Check Serverless Compute enabled +enabled, sbp_rec = getSecurityBestPracticeRecord(check_id, cloud_type) + +def dbsql_enable_serverless_compute(df): + if df is not None and not df.rdd.isEmpty(): + return (check_id, 0, {'enable_serverless_compute':'Serverless Compute enabled'} ) + else: + return (check_id, 1, {'enable_serverless_compute':'Serverless Compute not enabled'}) +if enabled: + tbl_name = 'global_temp.dbsql_workspaceconfig' + '_' + workspace_id + sql=f''' + SELECT enable_serverless_compute FROM + FROM {tbl_name} + WHERE enable_serverless_compute = true + ''' + sqlctrl(workspace_id, sql, dbsql_enable_serverless_compute) + + +# COMMAND ---------- + +check_id='62' # INFO-18 Check Delta Sharing CREATE_RECIPIENT and CREATE_SHARE permissions +enabled, sbp_rec = getSecurityBestPracticeRecord(check_id, cloud_type) + +def metastore_delta_sharing_permissions(df): + if df is not None and not df.rdd.isEmpty(): + uc_metasore = df.collect() + uc_metasore_dict = {num: [row.metastore_name,row.principal, row.privilege] for num,row in enumerate(uc_metasore)} + return (check_id, 0, uc_metasore_dict ) # intentionally kept the score to 0 as its not a pass or fail. Its more of FYI + else: + return (check_id, 0, {}) # intentionally kept the score to 0 as its not a pass or fail. Its more of FYI +if enabled: + tbl_name = 'global_temp.metastorepermissions' + '_' + workspace_id + sql=f''' + SELECT * FROM (SELECT metastore_name,principal,explode(privileges) as privilege + FROM {tbl_name} ) + WHERE privilege= "CREATE_RECIPIENT" OR privilege="CREATE_SHARE" + ''' + sqlctrl(workspace_id, sql, metastore_delta_sharing_permissions) + +# COMMAND ---------- + tcomp = time.time() - start_time print(f"Workspace Analysis - {tcomp} seconds to run") diff --git a/notebooks/Includes/workspace_settings.py b/notebooks/Includes/workspace_settings.py index be6c8379..287813d1 100644 --- a/notebooks/Includes/workspace_settings.py +++ b/notebooks/Includes/workspace_settings.py @@ -433,7 +433,7 @@ def enableNotebookGitVersioning(df): for row in df.rdd.collect(): value = row.value defn = {'defn' : row.defn.replace("'", '')} - if(value == 'true'): + if(value == None or value == 'true'): return (id, 0, defn) else: return (id, 1, defn) @@ -442,7 +442,7 @@ def enableNotebookGitVersioning(df): tbl_name = 'global_temp.workspacesettings' + '_' + workspace_id sql = f''' SELECT * FROM {tbl_name} - WHERE workspace_id = "{workspace_id}" AND name="enableNotebookGitVersioning" + WHERE name="enableNotebookGitVersioning" ''' sqlctrl(workspace_id, sql, enableNotebookGitVersioning) diff --git a/notebooks/Utils/common.py b/notebooks/Utils/common.py index cb11f39d..daad4892 100644 --- a/notebooks/Utils/common.py +++ b/notebooks/Utils/common.py @@ -358,3 +358,7 @@ def notifyworkspaceCompleted(workspaceID, completed): # COMMAND ---------- JSONLOCALTESTB = '{"account_id": "", "sql_warehouse_id": "4a936419ee9b9d68", "username_for_alerts": "sat@regemail", "verbosity": "info", "master_name_scope": "sat_scope", "master_name_key": "user", "master_pwd_scope": "sat_scope", "master_pwd_key": "pass", "workspace_pat_scope": "sat_scope", "workspace_pat_token_prefix": "sat_token", "dashboard_id": "317f4809-8d9d-4956-a79a-6eee51412217", "dashboard_folder": "../../dashboards/", "dashboard_tag": "SAT", "use_mastercreds": true, "subscription_id": "", "tenant_id": "", "client_id": "", "client_secret": "", "generate_pat_tokens": false, "url": "https://adb-83xxx7.17.azuredatabricks.net", "workspace_id": "83xxxx7", "clusterid": "0105-242242-ir40aiai", "sso": true, "scim": false, "object_storage_encryption": false, "vpc_peering": false, "table_access_control_enabled": false, "cloud_type":"azure"}' + +# COMMAND ---------- + + diff --git a/notebooks/Utils/workspace_bootstrap.py b/notebooks/Utils/workspace_bootstrap.py index 8468ebcc..8d8361b2 100644 --- a/notebooks/Utils/workspace_bootstrap.py +++ b/notebooks/Utils/workspace_bootstrap.py @@ -118,9 +118,9 @@ # COMMAND ---------- -from clientpkgs.db_sql_client import DBSqlClient +from clientpkgs.dbsql_client import DBSQLClient try: - db_sql_client = DBSqlClient(json_) + db_sql_client = DBSQLClient(json_) except Exception: loggr.exception("Exception encountered") @@ -130,6 +130,22 @@ # COMMAND ---------- +bootstrap('dbsql_alerts' + '_' + workspace_id, db_sql_client.get_alerts_list) + +# COMMAND ---------- + +bootstrap('dbsql_warehouselist' + '_' + workspace_id, db_sql_client.get_sql_warehouse_list) + +# COMMAND ---------- + +bootstrap('dbsql_warehouselistv2' + '_' + workspace_id, db_sql_client.get_sql_warehouse_listv2) + +# COMMAND ---------- + +bootstrap('dbsql_workspaceconfig' + '_' + workspace_id, db_sql_client.get_sql_workspace_config) + +# COMMAND ---------- + # MAGIC %md # MAGIC ##### IPAccessList @@ -395,6 +411,55 @@ # COMMAND ---------- +# MAGIC %md +# MAGIC ##### Unity Catalog + +# COMMAND ---------- + +from clientpkgs.unity_catalog_client import UnityCatalogClient +try: + uc_client = UnityCatalogClient(json_) +except: + loggr.exception("Exception encountered") + +# COMMAND ---------- + +bootstrap('unitycatalogmsv1' + '_' + workspace_id, uc_client.get_metastore_list) + +# COMMAND ---------- + +bootstrap('unitycatalogmsv2' + '_' + workspace_id, uc_client.get_workspace_metastore_assignments) + +# COMMAND ---------- + +bootstrap('unitycatalogexternallocations' + '_' + workspace_id, uc_client.get_external_locations) + +# COMMAND ---------- + +bootstrap('unitycatalogcredentials' + '_' + workspace_id, uc_client.get_credentials) + +# COMMAND ---------- + +bootstrap('unitycatalogshares' + '_' + workspace_id, uc_client.get_list_shares) + +# COMMAND ---------- + +bootstrap('unitycatalogshareproviders' + '_' + workspace_id, uc_client.get_sharing_providers_list) + +# COMMAND ---------- + +bootstrap('unitycatalogsharerecipients' + '_' + workspace_id, uc_client.get_sharing_recepients_list) + +# COMMAND ---------- + + bootstrap('unitycatalogcatlist' + '_' + workspace_id, uc_client.get_catalogs_list) + +# COMMAND ---------- + + bootstrap('metastorepermissions' + '_' + workspace_id, uc_client.get_grants_effective_permissions_ext) + +# COMMAND ---------- + # MAGIC %md # MAGIC ##### Workspace diff --git a/src/securityanalysistoolproject/clientpkgs/db_sql_client.py b/src/securityanalysistoolproject/clientpkgs/db_sql_client.py deleted file mode 100644 index f8482c4e..00000000 --- a/src/securityanalysistoolproject/clientpkgs/db_sql_client.py +++ /dev/null @@ -1,13 +0,0 @@ -'''dbsql module''' -from core.dbclient import SatDBClient - -class DBSqlClient(SatDBClient): - '''dbsql helper''' - - def get_sqlendpoint_list(self): - """ - Returns an array of json objects for jobruns. - """ - # fetch all jobsruns - endpoints_list = self.get("/sql/endpoints", version='2.0').get('endpoints', []) - return endpoints_list diff --git a/src/securityanalysistoolproject/clientpkgs/dbsql_client.py b/src/securityanalysistoolproject/clientpkgs/dbsql_client.py new file mode 100644 index 00000000..1a167b8f --- /dev/null +++ b/src/securityanalysistoolproject/clientpkgs/dbsql_client.py @@ -0,0 +1,50 @@ +'''DBSql Module''' +from core.dbclient import SatDBClient +import json + +class DBSQLClient(SatDBClient): + '''dbsql client helper''' + + def get_sqlendpoint_list(self): + """ + Returns an array of json objects for jobruns. + """ + # fetch all jobsruns + endpoints_list = self.get("/sql/endpoints", version='2.0').get('endpoints', []) + return endpoints_list + + def get_alerts_list(self): + """ + Returns an array of json objects for alerts. + """ + + alertslist = self.get("/preview/sql/alerts", version='2.0').get('elements', []) + return alertslist + + def get_sql_warehouse_list(self): + """ + Returns an array of json objects for sql warehouse. + """ + + sqlwarehouselist = self.get("/preview/sql/data_sources", version='2.0').get('elements', []) + return sqlwarehouselist + + def get_sql_warehouse_listv2(self): + """ + Returns an array of json objects for sql warehouse. + """ + sqlwarehousejson = self.get("/sql/warehouses", version='2.0') + sqlwarehouselist = [] + sqlwarehouselist.append(json.loads(json.dumps(sqlwarehousejson))) + return sqlwarehouselist + + + def get_sql_workspace_config(self): + """ + Returns an array of json objects for sql warehouse config. + """ + sqlworkspaceconfig = self.get("/sql/config/warehouses", version='2.0') + sqlconfiglist = [] + sqlconfiglist.append(json.loads(json.dumps(sqlworkspaceconfig))) + return sqlconfiglist + \ No newline at end of file diff --git a/src/securityanalysistoolproject/clientpkgs/unity_catalog_client.py b/src/securityanalysistoolproject/clientpkgs/unity_catalog_client.py new file mode 100644 index 00000000..c1cb6eb4 --- /dev/null +++ b/src/securityanalysistoolproject/clientpkgs/unity_catalog_client.py @@ -0,0 +1,175 @@ +'''unity catalog client module''' +from core.dbclient import SatDBClient +from core.logging_utils import LoggingUtils +import json + +LOGGR=None + +if LOGGR is None: + LOGGR = LoggingUtils.get_logger() + +class UnityCatalogClient(SatDBClient): + '''unity catalog helper''' + + def get_catalogs_list(self): + """ + Returns an array of json objects for catalogs list + """ + # fetch all catalogs list + catalogslist = self.get("/unity-catalog/catalogs", version='2.1').get('catalogs', []) + return catalogslist + + def get_schemas_list(self, catalogname): + """ + Returns list of schemas + """ + # fetch all schemaslist + schemaslist = self.get(f"/unity-catalog/schemas?catalog_name={catalogname}", version='2.1').get('schemas', []) + return schemaslist + + def get_tables(self, catalog_name, schema_name): + """ + Returns list of tables + """ + # fetch all schemaslist + query = f"/unity-catalog/tables?catalog_name={catalog_name}&schema_name={schema_name}" + tableslist = self.get(query, version='2.1').get('tables', []) + return tableslist + + def get_functions(self, catalog_name, schema_name): + """ + Returns an array of json objects for functions + """ + # fetch all functions + query = f"/unity-catalog/functions?catalog_name={catalog_name}&schema_name={schema_name}" + funcs = self.get(query, version='2.1').get('schemas', []) + return funcs + + def get_sharing_providers_list(self): + """ + Returns an array of json objects for sharing providers + """ + # fetch all sharing providers list + query = f"/unity-catalog/providers" + sharingproviderslist = self.get(query, version='2.1').get('providers', []) + return sharingproviderslist + + def get_sharing_recepients_list(self): + """ + Returns an array of json objects for sharing recepients + """ + # fetch all sharing recepients list + sharingrecepientslist = self.get("/unity-catalog/recipients", version='2.1').get('recipients', []) + return sharingrecepientslist + + def get_sharing_recepient_permissions(self, sharename): + """ + Returns an array of json objects for sharing recepients permission + """ + # fetch all acls list + sharingacl = self.get(f"/unity-catalog/recipients/{sharename}/share-permissions", version='2.1').get('permissions_out', []) + return sharingacl + + def get_list_shares(self): + """ + Returns an array of json objects for shares + """ + # fetch all shares + shareslist = self.get("/unity-catalog/shares", version='2.1').get('shares', []) + return shareslist + + def get_share_permissions(self, sharename): + """ + Returns an array of json objects for share permission + """ + # fetch all acls list + sharingacl = self.get(f"/unity-catalog/shares/{sharename}/permissions", version='2.1').get('privilege_assignments', []) + return sharingacl + + + def get_external_locations(self): + """ + Returns an array of json objects for external locations + """ + # fetch all external locations + extlocns = self.get("/unity-catalog/external-locations", version='2.1').get('external_locations', []) + return extlocns + + def get_workspace_metastore_assignments(self): + """ + Returns workspace metastore assignment + """ + # fetch all metastore assignment list + metastorejson = self.get("/unity-catalog/current-metastore-assignment", version='2.1') + metastoreassgnlist = [] + metastoreassgnlist.append(json.loads(json.dumps(metastorejson))) + return metastoreassgnlist + + + def get_workspace_metastore_summary(self): + """ + Returns workspace metastore summary + """ + # fetch all metastore assignment list + metastoresumjson = self.get("/unity-catalog/metastore_summary", version='2.1') + metastoresumlist = [] + metastoresumlist.append(json.loads(json.dumps(metastoresumjson))) + return metastoresumlist + + #Has to be an account admin to run this api + def get_metastore_list(self): + """ + Returns list of workspace metastore + """ + # fetch all metastores + # Exception: Error: GET request failed with code 403 {"error_code":"PERMISSION_DENIED","message":"Only account admin can list metastores.","details":[{"@type":"type.googleapis.com/google.rpc.RequestInfo","request_id":"b9353080-94ea-47b6-b551-083336de7d84","serving_data":""}] + try: + metastores = self.get("/unity-catalog/metastores", version='2.1').get('metastores', []) + except Exception as e: + LOGGR.exception(e) + return [] + return metastores + + def get_credentials(self): + """ + Returns list of credentials + """ + # fetch all schemaslist + credentialslist = self.get("/unity-catalog/storage-credentials", version='2.1').get('storage_credentials', []) + return credentialslist + + def get_grants_effective_permissions(self, securable_type, full_name): + """ + Returns effective permissions for securable type + :param securable_type like METASTORE, CATALOG, SCHEMA + :param full_name like metastore guid + """ + # fetch all schemaslist + permslist = self.get(f"/unity-catalog/permissions/{securable_type}/{full_name}", version='2.1').get('privilege_assignments', []) + return permslist + + def get_grants_permissions(self, securable_type, full_name): + """ + Returns permissions for securable type + :param securable_type like METASTORE, CATALOG, SCHEMA + :param full_name like metastore guid + """ + # fetch all schemaslist + permslist = self.get("/unity-catalog/effective-permissions/{securable_type}/{full_name}", version='2.1').get('privilege_assignments', []) + return permslist + + #the user should have account admin privileges + def get_grants_effective_permissions_ext(self): + arrperms=[] + arrlist = self.get_metastore_list() + for meta in arrlist: + metastore_id = meta['metastore_id'] + effperms = self.get_grants_effective_permissions('METASTORE', metastore_id) + for effpermselem in effperms: + effpermselem['metastore_id'] = meta['metastore_id'] + effpermselem['metastore_name'] = meta['name'] + arrperms.extend(effperms) + jsonarr = json.dumps(arrperms) + return arrperms + + \ No newline at end of file diff --git a/src/securityanalysistoolproject/setup.py b/src/securityanalysistoolproject/setup.py index da840066..bbf79935 100644 --- a/src/securityanalysistoolproject/setup.py +++ b/src/securityanalysistoolproject/setup.py @@ -5,7 +5,7 @@ DESCRIPTION = "Databricks Security Analysis Tool" -__version__ = "0.1.23" +__version__ = "0.1.24" this_directory = path.abspath(path.dirname(__file__)) with open(path.join(this_directory, "README.md"), encoding="utf-8") as f: diff --git a/src/securityanalysistoolproject/tests/conftest.py b/src/securityanalysistoolproject/tests/conftest.py index f9f9f409..bce2c946 100644 --- a/src/securityanalysistoolproject/tests/conftest.py +++ b/src/securityanalysistoolproject/tests/conftest.py @@ -13,6 +13,7 @@ def get_db_client(): configParser.read(configFilePath) jsonstr = configParser['MEISTERSTUFF']['json'] json_ = json.loads(jsonstr) + #workspace_id = json_['workspace_id'] LoggingUtils.set_logger_level(LoggingUtils.get_log_level(json_['verbosity'])) LOGGR = LoggingUtils.get_logger() diff --git a/src/securityanalysistoolproject/tests/readme.txt b/src/securityanalysistoolproject/tests/readme.txt index f7a09e03..2a19a555 100644 --- a/src/securityanalysistoolproject/tests/readme.txt +++ b/src/securityanalysistoolproject/tests/readme.txt @@ -1,3 +1,3 @@ #not able to make pytest work with visual studio code #run pytest -s from the terminal -# specific test pytest -s tests/test_funcs1.py::test_azure_cmk \ No newline at end of file +# specific test - pytest -s tests/test_funcs1.py::test_azure_cmk \ No newline at end of file diff --git a/src/securityanalysistoolproject/tests/test_dbsql.py b/src/securityanalysistoolproject/tests/test_dbsql.py new file mode 100644 index 00000000..89eb8793 --- /dev/null +++ b/src/securityanalysistoolproject/tests/test_dbsql.py @@ -0,0 +1,41 @@ +from core.logging_utils import LoggingUtils +from core import parser as pars +from core.dbclient import SatDBClient +from clientpkgs.dbsql_client import DBSQLClient + +def test_get_alerts_list(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + dbsqlobj = DBSQLClient(jsonstr) + settingsList = dbsqlobj.get_sqlendpoint_list() + print(settingsList) + +def test_get_alerts_list(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + dbsqlobj = DBSQLClient(jsonstr) + settingsList = dbsqlobj.get_alerts_list() + print(settingsList) + +def test_get_sql_warehouse_list(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + dbsqlobj = DBSQLClient(jsonstr) + settingsList = dbsqlobj.get_sql_warehouse_list() + print(settingsList) + +def test_get_sql_warehouse_listv2(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + dbsqlobj = DBSQLClient(jsonstr) + settingsList = dbsqlobj.get_sql_warehouse_listv2() + print(settingsList) + +def test_get_sql_workspace_config(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + dbsqlobj = DBSQLClient(jsonstr) + settingsList = dbsqlobj.get_sql_workspace_config() + print(settingsList) + + \ No newline at end of file diff --git a/src/securityanalysistoolproject/tests/test_uc.py b/src/securityanalysistoolproject/tests/test_uc.py new file mode 100644 index 00000000..045bd9d6 --- /dev/null +++ b/src/securityanalysistoolproject/tests/test_uc.py @@ -0,0 +1,132 @@ +from core.logging_utils import LoggingUtils +from core import parser as pars +from core.dbclient import SatDBClient +from clientpkgs.unity_catalog_client import UnityCatalogClient + +def test_get_catalogs_list(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + catalogslist = UnityCatalogClient(jsonstr) + settingsList = catalogslist.get_catalogs_list() + print(settingsList) + +def test_get_schemas_list(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + catalogslist = UnityCatalogClient(jsonstr) + sList = catalogslist.get_schemas_list('akangsha_catalog') + print(sList) + sList = catalogslist.get_schemas_list('nonexistentcat') + print(sList) + +def test_get_tables(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + catalogslist = UnityCatalogClient(jsonstr) + sList = catalogslist.get_tables('akangsha_catalog', 'akangsha_schema') + print(sList) + +def test_get_functions(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + catalogslist = UnityCatalogClient(jsonstr) + sList = catalogslist.get_functions('akangsha_catalog', 'akangsha_schema') + print(sList) + +def test_get_sharing_providers_list(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + catalogslist = UnityCatalogClient(jsonstr) + sList = catalogslist.get_sharing_providers_list() + print(sList) + +def test_get_sharing_recepients_list(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + catalogslist = UnityCatalogClient(jsonstr) + sList = catalogslist.get_sharing_recepients_list() + print(sList) + +def test_get_sharing_recepient_permissions(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + catalogslist = UnityCatalogClient(jsonstr) + sList = catalogslist.get_sharing_recepient_permissions('azure-field-eng-east') + print(sList) + +def test_get_list_shares(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + catalogslist = UnityCatalogClient(jsonstr) + sList = catalogslist.get_list_shares() + print(sList) + +def test_get_share_permissions(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + catalogslist = UnityCatalogClient(jsonstr) + sList = catalogslist.get_share_permissions('abc_to_mediacorp_share') + print(sList) + +def test_get_external_locations(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + catalogslist = UnityCatalogClient(jsonstr) + sList = catalogslist.get_external_locations() + print(sList) + + +def test_get_workspace_metastore_assignments(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + catalogslist = UnityCatalogClient(jsonstr) + sList = catalogslist.get_workspace_metastore_assignments() + print(sList) + +def test_get_workspace_metastore_summary(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + catalogslist = UnityCatalogClient(jsonstr) + sList = catalogslist.get_workspace_metastore_summary() + print(sList) + +def test_get_metastore_list(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + catalogslist = UnityCatalogClient(jsonstr) + sList = catalogslist.get_metastore_list() + print(sList) + +def test_credentials(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + catalogslist = UnityCatalogClient(jsonstr) + sList = catalogslist.get_credentials() + print(sList) + +def test_grants_permissions(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + catalogslist = UnityCatalogClient(jsonstr) + #guid of metastore + sList = catalogslist.get_grants_permissions('METASTORE', 'b169b504-4c54-49f2-bc3a-adf4b128f36d') + print('--------------------') + print(sList) + +def test_grants_effective_permissions(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + catalogslist = UnityCatalogClient(jsonstr) + #guid of metastore + sList = catalogslist.get_grants_effective_permissions('METASTORE', 'b169b504-4c54-49f2-bc3a-adf4b128f36d') + print('--------------------') + print(sList) + +def test_grants_effective_permissions_ext(get_db_client): + LOGGR = LoggingUtils.get_logger() + jsonstr = get_db_client + catalogslist = UnityCatalogClient(jsonstr) + #guid of metastore + sList = catalogslist.get_grants_effective_permissions_ext() + print('--------------------') + print(sList) \ No newline at end of file