feat: Centralized logs with structured JSON (#140)

Resolves #138
CloudSnorkel · Oct 1, 2024 · 78c35ee · 78c35ee
1 parent e493780
commit 78c35ee
Show file tree

Hide file tree

Showing 14 changed files with 424 additions and 1,032 deletions.
diff --git a/.projen/deps.json b/.projen/deps.json
diff --git a/.projenrc.js b/.projenrc.js
@@ -5,7 +5,7 @@ const project = new awscdk.AwsCdkConstructLibrary({
   author: 'Amir Szekely',
   authorAddress: 'amir@cloudsnorkel.com',
   stability: Stability.EXPERIMENTAL,
-  cdkVersion: '2.85.0', // for no more deprecated nodejs 14 in integration test
+  cdkVersion: '2.127.0', // 2.85.0 for no more deprecated nodejs 14 in integration test, 2.217.0 for lambda log settings
   defaultReleaseBranch: 'main',
   name: '@cloudsnorkel/cdk-rds-sanitized-snapshots',
   repositoryUrl: 'https://github.com/CloudSnorkel/cdk-rds-sanitized-snapshots.git',

diff --git a/package.json b/package.json
diff --git a/src/index.ts b/src/index.ts
@@ -6,6 +6,7 @@ import {
   aws_events_targets as events_targets,
   aws_iam as iam,
   aws_kms as kms,
+  aws_lambda as lambda,
   aws_logs as logs,
   aws_rds as rds,
   aws_stepfunctions as stepfunctions,
@@ -141,6 +142,7 @@ export class RdsSanitizedSnapshotter extends Construct {
   private readonly sqlScript: string;
   private readonly reencrypt: boolean;
   private readonly useExistingSnapshot: boolean;
+  private readonly logGroup: logs.ILogGroup;
 
   private readonly generalTags: {Key: string; Value: string}[];
   private readonly finalSnapshotTags: {Key: string; Value: string}[];
@@ -201,6 +203,11 @@ export class RdsSanitizedSnapshotter extends Construct {
     this.reencrypt = props.snapshotKey !== undefined;
     this.useExistingSnapshot = props.useExistingSnapshot ?? false;
 
+    this.logGroup = new logs.LogGroup(this, 'Logs', {
+      removalPolicy: cdk.RemovalPolicy.DESTROY,
+      retention: logs.RetentionDays.ONE_MONTH,
+    });
+
     this.dbClusterArn = cdk.Stack.of(this).formatArn({
       service: 'rds',
       resource: 'cluster',
@@ -329,7 +336,7 @@ export class RdsSanitizedSnapshotter extends Construct {
   }
 
   private dbParametersTask(databaseKey?: kms.IKey) {
-    const parametersFn = new ParametersFunction(this, 'parameters', { logRetention: logs.RetentionDays.ONE_MONTH });
+    const parametersFn = new ParametersFunction(this, 'parameters', { logGroup: this.logGroup, loggingFormat: lambda.LoggingFormat.JSON });
     const parametersState = new stepfunctions_tasks.LambdaInvoke(this, 'Get Parameters', {
       lambdaFunction: parametersFn,
       payload: stepfunctions.TaskInput.fromObject({
@@ -362,13 +369,15 @@ export class RdsSanitizedSnapshotter extends Construct {
 
   private findLatestSnapshot(id: string, databaseId: string) {
     const findFn = new FindSnapshotFunction(this, 'find-snapshot', {
-      logRetention: logs.RetentionDays.ONE_MONTH,
+      logGroup: this.logGroup,
+      loggingFormat: lambda.LoggingFormat.JSON,
       initialPolicy: [
         new iam.PolicyStatement({
           actions: ['rds:DescribeDBClusterSnapshots', 'rds:DescribeDBSnapshots'],
           resources: [this.dbClusterArn, this.dbInstanceArn],
         }),
       ],
+      timeout: cdk.Duration.minutes(1),
     });
 
     let payload = {
@@ -402,7 +411,8 @@ export class RdsSanitizedSnapshotter extends Construct {
 
   private waitForOperation(id: string, resourceType: 'snapshot' | 'cluster' | 'instance', databaseIdentifier: string, snapshotId?: string) {
     this.waitFn = this.waitFn ?? new WaitFunction(this, 'wait', {
-      logRetention: logs.RetentionDays.ONE_MONTH,
+      logGroup: this.logGroup,
+      loggingFormat: lambda.LoggingFormat.JSON,
       initialPolicy: [
         new iam.PolicyStatement({
           actions: ['rds:DescribeDBClusters', 'rds:DescribeDBClusterSnapshots', 'rds:DescribeDBSnapshots', 'rds:DescribeDBInstances'],
@@ -545,11 +555,6 @@ export class RdsSanitizedSnapshotter extends Construct {
   }
 
   private sanitize(): stepfunctions.IChainable {
-    const logGroup = new logs.LogGroup(this, 'Logs', {
-      removalPolicy: cdk.RemovalPolicy.DESTROY,
-      retention: logs.RetentionDays.ONE_MONTH,
-    });
-
     const mysqlTask = new ecs.FargateTaskDefinition(this, 'MySQL Task', {
       volumes: [{ name: 'config', host: {} }],
     });
@@ -561,7 +566,7 @@ export class RdsSanitizedSnapshotter extends Construct {
       image: ecs.AssetImage.fromRegistry('public.ecr.aws/docker/library/bash:4-alpine3.15'),
       command: ['bash', '-c', `echo "${mycnf}" > ~/.my.cnf && chmod 700 ~/.my.cnf`],
       logging: ecs.LogDriver.awsLogs({
-        logGroup,
+        logGroup: this.logGroup,
         streamPrefix: 'mysql-config',
       }),
       essential: false,
@@ -571,7 +576,7 @@ export class RdsSanitizedSnapshotter extends Construct {
       image: ecs.AssetImage.fromRegistry('public.ecr.aws/lts/mysql:latest'),
       command: ['mysql', '-e', this.sqlScript],
       logging: ecs.LogDriver.awsLogs({
-        logGroup,
+        logGroup: this.logGroup,
         streamPrefix: 'mysql-sanitize',
       }),
     });
@@ -583,7 +588,7 @@ export class RdsSanitizedSnapshotter extends Construct {
       image: ecs.AssetImage.fromRegistry('public.ecr.aws/lts/postgres:latest'),
       command: ['psql', '-c', this.sqlScript],
       logging: ecs.LogDriver.awsLogs({
-        logGroup,
+        logGroup: this.logGroup,
         streamPrefix: 'psql-sanitize',
       }),
     });
@@ -726,7 +731,8 @@ export class RdsSanitizedSnapshotter extends Construct {
 
   private deleteOldSnapshots(historyLimit: number) {
     const deleteOldFn = new DeleteOldFunction(this, 'delete-old', {
-      logRetention: logs.RetentionDays.ONE_MONTH,
+      logGroup: this.logGroup,
+      loggingFormat: lambda.LoggingFormat.JSON,
       timeout: cdk.Duration.minutes(5),
     });
     deleteOldFn.addToRolePolicy(new iam.PolicyStatement({

diff --git a/test/default.integ.snapshot/RDS-Sanitized-Snapshotter-RDS.assets.json b/test/default.integ.snapshot/RDS-Sanitized-Snapshotter-RDS.assets.json
@@ -1,15 +1,15 @@
 {
-  "version": "32.0.0",
+  "version": "36.0.0",
   "files": {
-    "43ed48878c85e3ce0324881d365f12acd6963f302748e21cd1257a713163350b": {
+    "da7ebb4e6882fcf053b199e0417772188b1369e252d27b7cf23ae1743467b6b0": {
       "source": {
         "path": "RDS-Sanitized-Snapshotter-RDS.template.json",
         "packaging": "file"
       },
       "destinations": {
         "current_account-current_region": {
           "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
-          "objectKey": "43ed48878c85e3ce0324881d365f12acd6963f302748e21cd1257a713163350b.json",
+          "objectKey": "da7ebb4e6882fcf053b199e0417772188b1369e252d27b7cf23ae1743467b6b0.json",
           "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
         }
       }

diff --git a/test/default.integ.snapshot/RDS-Sanitized-Snapshotter-RDS.template.json b/test/default.integ.snapshot/RDS-Sanitized-Snapshotter-RDS.template.json
@@ -78,27 +78,27 @@
     },
     "DeleteAutomatedBackups": true,
     "Engine": "mysql",
-    "MasterUsername": {
+    "MasterUserPassword": {
      "Fn::Join": [
       "",
       [
        "{{resolve:secretsmanager:",
        {
         "Ref": "MySQLInstanceSecret84563F6F"
        },
-       ":SecretString:username::}}"
+       ":SecretString:password::}}"
       ]
      ]
     },
-    "MasterUserPassword": {
+    "MasterUsername": {
      "Fn::Join": [
       "",
       [
        "{{resolve:secretsmanager:",
        {
         "Ref": "MySQLInstanceSecret84563F6F"
        },
-       ":SecretString:password::}}"
+       ":SecretString:username::}}"
       ]
      ]
     },
@@ -190,27 +190,27 @@
      "Ref": "MySQLClusterSubnets30A4ABD4"
     },
     "Engine": "aurora-mysql",
-    "MasterUsername": {
+    "MasterUserPassword": {
      "Fn::Join": [
       "",
       [
        "{{resolve:secretsmanager:",
        {
         "Ref": "MySQLClusterSecret06B35C31"
        },
-       ":SecretString:username::}}"
+       ":SecretString:password::}}"
       ]
      ]
     },
-    "MasterUserPassword": {
+    "MasterUsername": {
      "Fn::Join": [
       "",
       [
        "{{resolve:secretsmanager:",
        {
         "Ref": "MySQLClusterSecret06B35C31"
        },
-       ":SecretString:password::}}"
+       ":SecretString:username::}}"
       ]
      ]
     },
@@ -242,6 +242,7 @@
   "Key961B73FD": {
    "Type": "AWS::KMS::Key",
    "Properties": {
+    "Description": "RDS sanitize test source key",
     "KeyPolicy": {
      "Statement": [
       {
@@ -269,8 +270,7 @@
       }
      ],
      "Version": "2012-10-17"
-    },
-    "Description": "RDS sanitize test source key"
+    }
    },
    "UpdateReplacePolicy": "Delete",
    "DeletionPolicy": "Delete"
@@ -359,27 +359,27 @@
       "Arn"
      ]
     },
-    "MasterUsername": {
+    "MasterUserPassword": {
      "Fn::Join": [
       "",
       [
        "{{resolve:secretsmanager:",
        {
         "Ref": "PostgresInstanceSecret47B7DD5E"
        },
-       ":SecretString:username::}}"
+       ":SecretString:password::}}"
       ]
      ]
     },
-    "MasterUserPassword": {
+    "MasterUsername": {
      "Fn::Join": [
       "",
       [
        "{{resolve:secretsmanager:",
        {
         "Ref": "PostgresInstanceSecret47B7DD5E"
        },
-       ":SecretString:password::}}"
+       ":SecretString:username::}}"
       ]
      ]
     },
@@ -478,27 +478,27 @@
       "Arn"
      ]
     },
-    "MasterUsername": {
+    "MasterUserPassword": {
      "Fn::Join": [
       "",
       [
        "{{resolve:secretsmanager:",
        {
         "Ref": "PostgresClusterSecretEB353FC9"
        },
-       ":SecretString:username::}}"
+       ":SecretString:password::}}"
       ]
      ]
     },
-    "MasterUserPassword": {
+    "MasterUsername": {
      "Fn::Join": [
       "",
       [
        "{{resolve:secretsmanager:",
        {
         "Ref": "PostgresClusterSecretEB353FC9"
        },
-       ":SecretString:password::}}"
+       ":SecretString:username::}}"
       ]
      ]
     },

diff --git a/test/default.integ.snapshot/RDS-Sanitized-Snapshotter-SFN.assets.json b/test/default.integ.snapshot/RDS-Sanitized-Snapshotter-SFN.assets.json
@@ -1,5 +1,5 @@
 {
-  "version": "32.0.0",
+  "version": "36.0.0",
   "files": {
     "73ae9c6df09ad4bfbc13c9d4e1f9695a2de8da41bbb8b0037182a8cda9a710c6": {
       "source": {
@@ -14,19 +14,6 @@
         }
       }
     },
-    "5fa1330271b8967d9254ba2d4a07144f8acefe8b77e6d6bba38261373a50d5f8": {
-      "source": {
-        "path": "asset.5fa1330271b8967d9254ba2d4a07144f8acefe8b77e6d6bba38261373a50d5f8",
-        "packaging": "zip"
-      },
-      "destinations": {
-        "current_account-current_region": {
-          "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
-          "objectKey": "5fa1330271b8967d9254ba2d4a07144f8acefe8b77e6d6bba38261373a50d5f8.zip",
-          "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
-        }
-      }
-    },
     "66486f7e33c34ceaae0d26eda8231c31f462018de9b6f34e598b3cc0df48b44f": {
       "source": {
         "path": "asset.66486f7e33c34ceaae0d26eda8231c31f462018de9b6f34e598b3cc0df48b44f.lambda",
@@ -40,15 +27,15 @@
         }
       }
     },
-    "4d342d6a3f6400ba76ff90f4cd6140b4d7b6f7a8a8c14189a2b75f634544caac": {
+    "3961ed00ffe563e995f9d7e4fa79705f773ae1376c3cd5b54e937b4bb5f047c9": {
       "source": {
         "path": "RDS-Sanitized-Snapshotter-SFN.template.json",
         "packaging": "file"
       },
       "destinations": {
         "current_account-current_region": {
           "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
-          "objectKey": "4d342d6a3f6400ba76ff90f4cd6140b4d7b6f7a8a8c14189a2b75f634544caac.json",
+          "objectKey": "3961ed00ffe563e995f9d7e4fa79705f773ae1376c3cd5b54e937b4bb5f047c9.json",
           "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
         }
       }