Resolve GitHub comments

Crown-Commercial-Service · Oct 16, 2024 · 1957f4a · 1957f4a
1 parent 1ac8e13
commit 1957f4a
Show file tree

Hide file tree

Showing 6 changed files with 146 additions and 136 deletions.
diff --git a/modules/postgres-etl/extract/extract_iam.tf b/modules/postgres-etl/extract/extract_iam.tf
@@ -23,8 +23,8 @@ data "aws_iam_policy_document" "s3" {
     ]
 
     resources = [
-      "arn:aws:s3:::ccs-digitalmarketplace-postgres-etl-extract-${var.environment_name}",
-      "arn:aws:s3:::ccs-digitalmarketplace-postgres-etl-extract-${var.environment_name}/*"
+      "arn:aws:s3:::ccs-digitalmarketplace-${var.migrator_name}-extract-${var.environment_name}",
+      "arn:aws:s3:::ccs-digitalmarketplace-${var.migrator_name}-extract-${var.environment_name}/*"
     ]
   }
 }
@@ -54,18 +54,18 @@ data "aws_iam_policy_document" "ecr" {
     actions = [
       "ecr:BatchCheckLayerAvailability",
       "ecr:BatchGetImage",
-      "ecr:DescribeImages", # Possibly not needed
+      # "ecr:DescribeImages", # Possibly not needed
       "ecr:GetDownloadUrlForLayer",
-      "ecr:ListImages", # Possibly not needed
+      # "ecr:ListImages", # Possibly not needed
     ]
 
     resources = [
-      "arn:aws:ecr:${var.aws_region}:473251818902:repository/postgres-etl",   # Dev
-      "arn:aws:ecr:${var.aws_region}:473251818902:repository/postgres-etl:*", # Dev
-      "arn:aws:ecr:${var.aws_region}:665505400356:repository/postgres-etl",   # SBX
-      "arn:aws:ecr:${var.aws_region}:665505400356:repository/postgres-etl:*", # SBX
-      "arn:aws:ecr:${var.aws_region}:974531504241:repository/postgres-etl",   # Prod
-      "arn:aws:ecr:${var.aws_region}:974531504241:repository/postgres-etl:*"  # Prod
+      "arn:aws:ecr:${var.aws_region}:473251818902:repository/${var.migrator_name}",   # Dev
+      "arn:aws:ecr:${var.aws_region}:473251818902:repository/${var.migrator_name}:*", # Dev
+      "arn:aws:ecr:${var.aws_region}:665505400356:repository/${var.migrator_name}",   # SBX
+      "arn:aws:ecr:${var.aws_region}:665505400356:repository/${var.migrator_name}:*", # SBX
+      "arn:aws:ecr:${var.aws_region}:974531504241:repository/${var.migrator_name}",   # Prod
+      "arn:aws:ecr:${var.aws_region}:974531504241:repository/${var.migrator_name}:*"  # Prod
     ]
   }
 }
@@ -74,7 +74,7 @@ data "aws_iam_policy_document" "ecs_exec" {
   version = "2012-10-17"
 
   statement {
-    sid = "AllowECSExecPolicy"
+    sid = "AllowECSExec"
 
     effect = "Allow"
 
@@ -123,7 +123,7 @@ data "aws_iam_policy_document" "rds_to_s3_sfn" {
     ]
 
     resources = [
-      var.ecs_extract_execution_role.arn, # Do we need to pass the execution role?
+      # var.ecs_extract_execution_role.arn, # Do we need to pass the execution role?
       module.extract_task.task_role_arn
     ]
   }

diff --git a/modules/postgres-etl/extract/extract_s3.tf b/modules/postgres-etl/extract/extract_s3.tf
@@ -28,12 +28,17 @@ resource "aws_s3_bucket_policy" "extract" {
         "Effect" : "Allow",
         "Principal" : {
           "AWS" : [
-            "arn:aws:iam::473251818902:role/eks-paas-postgres-etl", # Dev
-            "arn:aws:iam::665505400356:role/eks-paas-postgres-etl", # SBX
-            "arn:aws:iam::974531504241:role/eks-paas-postgres-etl"  # PROD
+            "arn:aws:iam::473251818902:role/eks-paas-${var.migrator_name}", # Dev
+            "arn:aws:iam::665505400356:role/eks-paas-${var.migrator_name}", # SBX
+            "arn:aws:iam::974531504241:role/eks-paas-${var.migrator_name}"  # PROD
           ]
         },
-        "Action" : "s3:*", # Adjust later
+        "Action" : [
+          "s3:GetObject",
+          "s3:PutObject",
+          "s3:DeleteObject",
+          "s3:ListBucket"
+        ],
         "Resource" : [
           "${aws_s3_bucket.extract.arn}",
           "${aws_s3_bucket.extract.arn}/*"

diff --git a/modules/postgres-etl/extract/extract_sqs.tf b/modules/postgres-etl/extract/extract_sqs.tf
@@ -33,7 +33,7 @@ data "aws_iam_policy_document" "extract_sqs" {
     principals {
       type = "AWS"
       identifiers = [
-        "arn:aws:iam::665505400356:role/eks-paas-mountpoint-s3-csi-driver"
+        "arn:aws:iam::665505400356:role/eks-paas-${var.migrator_name}"
       ]
     }
 
@@ -42,13 +42,13 @@ data "aws_iam_policy_document" "extract_sqs" {
     ]
 
     resources = [
-      "arn:aws:sqs:*:*:postgres-etl-s3"
+      "arn:aws:sqs:*:*:${var.migrator_name}-s3"
     ]
   }
 }
 
 resource "aws_sqs_queue" "extract" {
-  name = "postgres-etl-s3"
+  name = "${var.migrator_name}-s3"
 
   policy = data.aws_iam_policy_document.extract_sqs.json
 
@@ -61,5 +61,5 @@ resource "aws_sqs_queue" "extract" {
 }
 
 resource "aws_sqs_queue" "extract_dlq" {
-  name = "postgres-etl-s3-dlq"
+  name = "${var.migrator_name}-s3-dlq"
 }
diff --git a/modules/postgres-etl/load/load_iam.tf b/modules/postgres-etl/load/load_iam.tf
@@ -23,8 +23,8 @@ data "aws_iam_policy_document" "s3" {
     ]
 
     resources = [
-      "arn:aws:s3:::ccs-digitalmarketplace-postgres-etl-load-${var.environment_name}",
-      "arn:aws:s3:::ccs-digitalmarketplace-postgres-etl-load-${var.environment_name}/*"
+      "arn:aws:s3:::ccs-digitalmarketplace-${var.migrator_name}-load-${var.environment_name}",
+      "arn:aws:s3:::ccs-digitalmarketplace-${var.migrator_name}-load-${var.environment_name}/*"
     ]
   }
 }
@@ -54,18 +54,18 @@ data "aws_iam_policy_document" "ecr" {
     actions = [
       "ecr:BatchCheckLayerAvailability",
       "ecr:BatchGetImage",
-      "ecr:DescribeImages",
+      # "ecr:DescribeImages", # Possibly not needed
       "ecr:GetDownloadUrlForLayer",
-      "ecr:ListImages",
+      # "ecr:ListImages", # Possibly not needed
     ]
 
     resources = [
-      "arn:aws:ecr:${var.aws_region}:473251818902:repository/postgres-etl",   # Dev
-      "arn:aws:ecr:${var.aws_region}:473251818902:repository/postgres-etl:*", # Dev
-      "arn:aws:ecr:${var.aws_region}:665505400356:repository/postgres-etl",   # SBX
-      "arn:aws:ecr:${var.aws_region}:665505400356:repository/postgres-etl:*", # SBX
-      "arn:aws:ecr:${var.aws_region}:974531504241:repository/postgres-etl",   # Prod
-      "arn:aws:ecr:${var.aws_region}:974531504241:repository/postgres-etl:*"  # Prod
+      "arn:aws:ecr:${var.aws_region}:473251818902:repository/${var.migrator_name}",   # Dev
+      "arn:aws:ecr:${var.aws_region}:473251818902:repository/${var.migrator_name}:*", # Dev
+      "arn:aws:ecr:${var.aws_region}:665505400356:repository/${var.migrator_name}",   # SBX
+      "arn:aws:ecr:${var.aws_region}:665505400356:repository/${var.migrator_name}:*", # SBX
+      "arn:aws:ecr:${var.aws_region}:974531504241:repository/${var.migrator_name}",   # Prod
+      "arn:aws:ecr:${var.aws_region}:974531504241:repository/${var.migrator_name}:*"  # Prod
     ]
   }
 }
@@ -74,7 +74,7 @@ data "aws_iam_policy_document" "ecs_exec" {
   version = "2012-10-17"
 
   statement {
-    sid = "AllowECSExecPolicy"
+    sid = "AllowECSExec"
 
     effect = "Allow"
 
@@ -130,6 +130,72 @@ data "aws_iam_policy_document" "read_creds_ssm" {
   }
 }
 
+data "aws_iam_policy_document" "s3_to_rds_sfn" {
+
+  version = "2012-10-17"
+
+  statement {
+    sid = "AllowPassEcsExecRole"
+
+    effect = "Allow"
+
+    actions = [
+      "iam:GetRole",
+      "iam:PassRole"
+    ]
+
+    resources = [
+      # var.ecs_load_execution_role.arn, # Do we need to pass the execution role?
+      module.load_task.task_role_arn
+    ]
+  }
+
+  statement {
+    sid = "AllowRunPGETLTasks"
+
+    effect = "Allow"
+
+    actions = [
+      "ecs:RunTask"
+    ]
+
+    resources = [
+      "${module.load_task.task_definition_arn_without_revision}:*"
+    ]
+  }
+
+  statement {
+    sid = "AllowStopPGETLTasks"
+
+    effect = "Allow"
+
+    actions = [
+      "ecs:DescribeTasks",
+      "ecs:StopTask"
+    ]
+
+    resources = [
+      "*"
+    ]
+  }
+
+  statement {
+    sid = "AllowDotSyncExecutionOfEcsTasks"
+
+    effect = "Allow"
+
+    actions = [
+      "events:DescribeRule",
+      "events:PutRule",
+      "events:PutTargets"
+    ]
+
+    resources = [
+      "arn:aws:events:${var.aws_region}:${var.aws_account_id}:rule/StepFunctionsGetEventsForECSTaskRule"
+    ]
+  }
+}
+
 resource "aws_iam_policy" "k8s_trigger_sfn" {
   name        = "k8s-trigger-sfn"
   description = "Allows the k8s-postgres-etl role to trigger the Postgres ETL Step Function"
@@ -146,25 +212,42 @@ resource "aws_iam_policy" "k8s_trigger_sfn" {
 }
 
 resource "aws_iam_role" "k8s_postgres_etl" {
-  name = "k8s-postgres-etl"
+  name = "k8s-${var.migrator_name}"
   assume_role_policy = jsonencode({
     Version = "2012-10-17",
     Statement = [
       {
         Effect = "Allow",
         Principal = {
-          AWS = "arn:aws:iam::665505400356:role/eks-paas-mountpoint-s3-csi-driver"
+          AWS = "arn:aws:iam::665505400356:role/eks-paas-${var.migrator_name}"
         },
         Action = "sts:AssumeRole"
       }
     ]
   })
 }
 
-resource "aws_iam_role_policy" "s3__postgres_etl_load" {
-  name   = "${var.migrator_name}-s3-load"
-  role   = module.load_task.task_role_name
-  policy = data.aws_iam_policy_document.s3.json
+resource "aws_iam_role" "s3_to_rds_sfn" {
+
+  name = "${var.migrator_name}-s3-to-rds-sfn"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+
+        Effect = "Allow"
+
+        Sid = "AllowStatesAssumeRole"
+
+        Principal = {
+          Service = "states.amazonaws.com"
+        }
+      }
+    ]
+  })
 }
 
 resource "aws_iam_role_policy" "ecr" {
@@ -191,6 +274,17 @@ resource "aws_iam_role_policy" "logging" {
   policy = data.aws_iam_policy_document.logging.json
 }
 
+resource "aws_iam_role_policy" "s3_to_rds_sfn" {
+  role   = aws_iam_role.s3_to_rds_sfn.name
+  policy = data.aws_iam_policy_document.s3_to_rds_sfn.json
+}
+
+resource "aws_iam_role_policy" "s3__postgres_etl_load" {
+  name   = "${var.migrator_name}-s3-load"
+  role   = module.load_task.task_role_name
+  policy = data.aws_iam_policy_document.s3.json
+}
+
 resource "aws_iam_role_policy_attachment" "k8s_etl_trigger_sfn" {
   role       = aws_iam_role.k8s_postgres_etl.name
   policy_arn = aws_iam_policy.k8s_trigger_sfn.arn

diff --git a/modules/postgres-etl/load/load_s3.tf b/modules/postgres-etl/load/load_s3.tf
@@ -18,12 +18,17 @@ resource "aws_s3_bucket_policy" "load" {
         "Effect" : "Allow",
         "Principal" : {
           "AWS" : [
-            "arn:aws:iam::473251818902:role/eks-paas-postgres-etl", # Dev
-            "arn:aws:iam::665505400356:role/eks-paas-postgres-etl", # SBX
-            "arn:aws:iam::974531504241:role/eks-paas-postgres-etl"  # PROD
+            "arn:aws:iam::473251818902:role/eks-paas-${var.migrator_name}", # Dev
+            "arn:aws:iam::665505400356:role/eks-paas-${var.migrator_name}", # SBX
+            "arn:aws:iam::974531504241:role/eks-paas-${var.migrator_name}"  # PROD
           ]
         },
-        "Action" : "s3:*", # Adjust later
+        "Action" : [
+          "s3:GetObject",
+          "s3:PutObject",
+          "s3:DeleteObject",
+          "s3:ListBucket"
+        ],
         "Resource" : [
           "${aws_s3_bucket.load.arn}",
           "${aws_s3_bucket.load.arn}/*"