From a7c781b7c03a30b33e04c6c09b26321ea100da3c Mon Sep 17 00:00:00 2001 From: xpk Date: Fri, 12 Jan 2024 13:49:50 +0800 Subject: [PATCH] NEW: s3 replication module --- .../storage/s3-bucket-replication/README.md | 50 ++++ modules/storage/s3-bucket-replication/main.tf | 131 ++++++++++ .../storage/s3-bucket-replication/outputs.tf | 3 + .../s3-bucket-replication/variables.tf | 20 ++ .../storage/s3-bucket-replication/versions.tf | 10 + modules/storage/s3_bucket_2023/README.md | 226 +++++++++++++++++- modules/storage/s3_bucket_2023/main.tf | 72 +++++- modules/storage/s3_bucket_2023/variables.tf | 9 + 8 files changed, 516 insertions(+), 5 deletions(-) create mode 100644 modules/storage/s3-bucket-replication/README.md create mode 100644 modules/storage/s3-bucket-replication/main.tf create mode 100644 modules/storage/s3-bucket-replication/outputs.tf create mode 100644 modules/storage/s3-bucket-replication/variables.tf create mode 100644 modules/storage/s3-bucket-replication/versions.tf diff --git a/modules/storage/s3-bucket-replication/README.md b/modules/storage/s3-bucket-replication/README.md new file mode 100644 index 0000000..8904c0a --- /dev/null +++ b/modules/storage/s3-bucket-replication/README.md @@ -0,0 +1,50 @@ + +## Requirements + +| Name | Version | +|------|---------| +| terraform | >= 1.3.0 | +| aws | >= 5.0 | + +## Providers + +| Name | Version | +|------|---------| +| aws | >= 5.0 | +| random | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_role.replication-role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.role-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_s3_bucket_replication_configuration.replication-config](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_replication_configuration) | resource | +| [random_id.rid](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.replication-role-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_s3_bucket.destination-bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/s3_bucket) | data source | +| [aws_s3_bucket.source-bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/s3_bucket) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|------------------|:--------:| +| destination-bucket-account-id | Account id of destination bucket. | `string` | `"111122223333"` | no | +| destination-bucket-encryption-key-arn | Encryption key arn for destination bucket | `string` | n/a | yes | +| destination-bucket-name | Name of destination bucket | `string` | n/a | yes | +| source-bucket-name | Name of source s3 bucket | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| replication-role-arn | n/a | + +--- +## Authorship +This module was developed by xpk. \ No newline at end of file diff --git a/modules/storage/s3-bucket-replication/main.tf b/modules/storage/s3-bucket-replication/main.tf new file mode 100644 index 0000000..841e9d9 --- /dev/null +++ b/modules/storage/s3-bucket-replication/main.tf @@ -0,0 +1,131 @@ +# sets up data sources for s3 buckets + +data "aws_s3_bucket" "source-bucket" { + bucket = var.source-bucket-name +} + +data "aws_s3_bucket" "destination-bucket" { + bucket = var.destination-bucket-name +} + +# Create replication role in source account +data "aws_iam_policy_document" "assume_role_policy" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["s3.amazonaws.com"] + } + } +} + +data "aws_iam_policy_document" "replication-role-policy" { + statement { + sid = "AccessToReplicaBucket" + actions = [ + "s3:ReplicateObject", + "s3:ReplicateDelete", + "s3:ReplicateTags", + "s3:ObjectOwnerOverrideToBucketOwner" + ] + effect = "Allow" + resources = [ + data.aws_s3_bucket.source-bucket.arn, + data.aws_s3_bucket.destination-bucket.arn, + "${data.aws_s3_bucket.source-bucket.arn}/*", + "${data.aws_s3_bucket.destination-bucket.arn}/*" + ] + } + statement { + sid = "ReadAccessOnSourceBuckets" + actions = ["s3:Get*", "s3:List*"] + effect = "Allow" + resources = [ + data.aws_s3_bucket.source-bucket.arn, + ] + } + statement { + sid = "ObjectAccessOnSourceBuckets" + actions = [ + "s3:GetObjectVersionForReplication", + "s3:GetObjectVersionAcl", + "s3:GetObjectVersionTagging" + ] + effect = "Allow" + resources = [ + "${data.aws_s3_bucket.source-bucket.arn}/*" + ] + } + statement { + sid = "DecryptSourceBucketObjects" + actions = [ + "kms:Decrypt" + ] + effect = "Allow" + resources = ["*"] + } + statement { + sid = "EncryptReplicaObjects" + actions = [ + "kms:Encrypt" + ] + effect = "Allow" + resources = ["*"] + } +} + +resource "random_id" "rid" { + byte_length = 4 +} + +resource "aws_iam_role" "replication-role" { + name = "BucketReplicationRole${random_id.rid.dec}" + assume_role_policy = data.aws_iam_policy_document.assume_role_policy.json +} + +resource "aws_iam_role_policy" "role-policy" { + name = "bucket-replication" + role = aws_iam_role.replication-role.id + policy = data.aws_iam_policy_document.replication-role-policy.json +} + +# Setup bucket replication +resource "aws_s3_bucket_replication_configuration" "replication-config" { + role = aws_iam_role.replication-role.arn + bucket = var.source-bucket-name + + rule { + id = "ReplicateAll" + + status = "Enabled" + + source_selection_criteria { + sse_kms_encrypted_objects { + status = "Enabled" + } + } + + # V2 replication configurations + delete_marker_replication { + status = "Enabled" + } + + filter { + } + + destination { + bucket = data.aws_s3_bucket.destination-bucket.arn + account = var.destination-bucket-account-id + access_control_translation { + owner = "Destination" + } + encryption_configuration { + replica_kms_key_id = var.destination-bucket-encryption-key-arn + } + metrics { + status = "Enabled" + } + } + } +} diff --git a/modules/storage/s3-bucket-replication/outputs.tf b/modules/storage/s3-bucket-replication/outputs.tf new file mode 100644 index 0000000..fd0bfde --- /dev/null +++ b/modules/storage/s3-bucket-replication/outputs.tf @@ -0,0 +1,3 @@ +output replication-role-arn { + value = aws_iam_role.replication-role.arn +} \ No newline at end of file diff --git a/modules/storage/s3-bucket-replication/variables.tf b/modules/storage/s3-bucket-replication/variables.tf new file mode 100644 index 0000000..f58caf0 --- /dev/null +++ b/modules/storage/s3-bucket-replication/variables.tf @@ -0,0 +1,20 @@ +variable source-bucket-name { + type = string + description = "Name of source s3 bucket" +} + +variable destination-bucket-name { + type = string + description = "Name of destination bucket" +} + +variable destination-bucket-account-id { + type = string + description = "Account id of destination bucket. Defaults to BEA-SYS-LOG-UAT" + default = "894849410890" +} + +variable destination-bucket-encryption-key-arn { + type = string + description = "Encryption key arn for destination bucket" +} \ No newline at end of file diff --git a/modules/storage/s3-bucket-replication/versions.tf b/modules/storage/s3-bucket-replication/versions.tf new file mode 100644 index 0000000..f0f3f05 --- /dev/null +++ b/modules/storage/s3-bucket-replication/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.3.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } +} diff --git a/modules/storage/s3_bucket_2023/README.md b/modules/storage/s3_bucket_2023/README.md index 279eaa7..a4ccbd9 100644 --- a/modules/storage/s3_bucket_2023/README.md +++ b/modules/storage/s3_bucket_2023/README.md @@ -36,4 +36,228 @@ module "bucket1" { noncurrent_version_expiration_days = 731 } -``` \ No newline at end of file +``` + +## Note on bucket replication +To securely replicate a bucket to a bucket in another aws account, kms key is required. + +Steps to setup replication are: +1. Create replication iam role on the source account, with an assume role policy trusting s3 +```json + { + "Effect":"Allow", + "Principal":{ + "Service":"s3.amazonaws.com" + }, + "Action":"sts:AssumeRole" + } +``` +The role needs permissions granted in the role iam policy. For example: +```json +{ + "Statement": [ + { + "Action": [ + "s3:ListBucket", + "s3:GetReplicationConfiguration" + ], + "Effect": "Allow", + "Resource": "arn:aws:s3:::whk1-bea-icc-mbk-prd-vpc01-flowlog-s3-accept", + "Sid": "" + }, + { + "Action": [ + "s3:GetObjectVersionTagging", + "s3:GetObjectVersionForReplication", + "s3:GetObjectVersionAcl" + ], + "Effect": "Allow", + "Resource": "arn:aws:s3:::whk1-bea-icc-mbk-prd-vpc01-flowlog-s3-accept/*", + "Sid": "" + }, + { + "Action": [ + "s3:ReplicateTags", + "s3:ReplicateObject", + "s3:ReplicateDelete", + "s3:ObjectOwnerOverrideToBucketOwner" + ], + "Effect": "Allow", + "Resource": "arn:aws:s3:::whk1-bea-icc-log-mbk-prd-vpc01-flowlog-s3-accept/*", + "Sid": "" + }, + { + "Effect": "Allow", + "Action": [ + "kms:Decrypt", + "kms:GenerateDataKey" + ], + "Resource": [ + "arn:aws:kms:ap-east-1:851239346925:key/708b6ece-05f5-40ed-a91c-dbcf2af46407" + ] + }, + { + "Effect": "Allow", + "Action": [ + "kms:GenerateDataKey", + "kms:Encrypt" + ], + "Resource": [ + "arn:aws:kms:ap-east-1:894849410890:key/b555d9d6-d451-4ec8-8ca2-cb6849cadee4" + ] + } + ], + "Version": "2012-10-17" +} +``` +If bucket key is used, then additional permission needs to be granted +```json +{ + "Action":[ + "kms:Decrypt" + ], + "Effect":"Allow", + "Condition":{ + "StringLike":{ + "kms:ViaService":"s3.ap-east-1.amazonaws.com", + "kms:EncryptionContext:aws:s3:arn":[ + "arn:aws:s3:::/*" + ] + } + }, + "Resource":[ + "arn:aws:kms:ap-east-1::key/" + ] + }, + { + "Action":[ + "kms:Encrypt" + ], + "Effect":"Allow", + "Condition":{ + "StringLike":{ + "kms:ViaService":"s3.ap-east-1.amazonaws.com", + "kms:EncryptionContext:aws:s3:arn":[ + "arn:aws:s3:::/*" + ] + } + }, + "Resource":[ + "arn:aws:kms:ap-east-1::key/" + ] + } +``` + +2. On the destination account, grant access in KMS key policy +```json +{ + "Version": "2012-10-17", + "Id": "key-consolepolicy-3", + "Statement": [ + { + "Sid": "Enable IAM User Permissions", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam:::root" + }, + "Action": "kms:*", + "Resource": "*" + }, + { + "Sid": "Allow use of the key", + "Effect": "Allow", + "Principal": { + "AWS": [ + "arn:aws:iam:::root", + "arn:aws:iam:::root" + ] + }, + "Action": [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:DescribeKey" + ], + "Resource": "*" + }, + { + "Sid": "Allow attachment of persistent resources", + "Effect": "Allow", + "Principal": { + "AWS": [ + "arn:aws:iam:::root", + "arn:aws:iam:::root" + ] + }, + "Action": [ + "kms:CreateGrant", + "kms:ListGrants", + "kms:RevokeGrant" + ], + "Resource": "*", + "Condition": { + "Bool": { + "kms:GrantIsForAWSResource": "true" + } + } + }, + { + "Sid": "Allow AWS Service to use the key", + "Effect": "Allow", + "Principal": { + "Service": [ + "s3.amazonaws.com", + "delivery.logs.amazonaws.com", + "cloudtrail.amazonaws.com" + ] + }, + "Action": [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:DescribeKey" + ], + "Resource": "*" + } + ] +} +``` + +3. Edit destination bucket policy +```json +{ + "Version": "2012-10-17", + "Id": "", + "Statement": [ + { + "Sid": "Set permissions for objects", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam:::root" + }, + "Action": [ + "s3:ReplicateDelete", + "s3:ReplicateObject", + "s3:ReplicateTags", + "s3:ObjectOwnerOverrideToBucketOwner" + ], + "Resource": "arn:aws:s3:::/*" + }, + { + "Sid": "Set permissions on bucket", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam:::root" + }, + "Action": [ + "s3:List*", + "s3:GetBucketVersioning", + "s3:PutBucketVersioning" + ], + "Resource": "arn:aws:s3:::" + } + ] +} +``` diff --git a/modules/storage/s3_bucket_2023/main.tf b/modules/storage/s3_bucket_2023/main.tf index b075b94..3906bf9 100644 --- a/modules/storage/s3_bucket_2023/main.tf +++ b/modules/storage/s3_bucket_2023/main.tf @@ -11,9 +11,34 @@ resource "aws_s3_bucket_public_access_block" "block_public_access" { restrict_public_buckets = true } +# Add SecureTransport restriction by default +data "aws_iam_policy_document" "bucket_policy" { + source_policy_documents = [var.bucket_policy_json] + + statement { + sid = "AllowSSLRequestsOnly" + actions = ["s3:*"] + effect = "Deny" + principals { + type = "*" + identifiers = ["*"] + } + resources = [ + aws_s3_bucket.this.arn, + "${aws_s3_bucket.this.arn}/*" + ] + condition { + test = "Bool" + values = [false] + variable = "aws:SecureTransport" + } + } +} + resource "aws_s3_bucket_policy" "bucket_policy" { bucket = aws_s3_bucket.this.id - policy = var.bucket_policy_json + # policy = var.bucket_policy_json + policy = data.aws_iam_policy_document.bucket_policy.json } resource "aws_s3_bucket_lifecycle_configuration" "lifecycle" { @@ -93,15 +118,54 @@ resource "aws_s3_bucket_versioning" "versioning" { } resource "aws_s3_bucket_replication_configuration" "replication" { - count = var.enable_replication && var.enable_versioning ? 1 : 0 - role = var.replication_role_arn - bucket = aws_s3_bucket.this.id + count = var.enable_replication && var.enable_versioning ? 1 : 0 + role = var.replication_role_arn + bucket = aws_s3_bucket.this.id + + rule { id = "replrule1" status = "Enabled" + delete_marker_replication { + status = "Enabled" + } + + source_selection_criteria { + replica_modifications { + status = "Enabled" + } + sse_kms_encrypted_objects { + status = "Enabled" + } + } + destination { bucket = var.replication_dest_bucket_name storage_class = "INTELLIGENT_TIERING" + + account = var.replication_destination_aws_account_id + + encryption_configuration { + replica_kms_key_id = var.replication_destination_kms_key_arn + } + + access_control_translation { + owner = "Destination" + } + + replication_time { + status = "Enabled" + time { + minutes = 15 + } + } + + metrics { + status = "Enabled" + event_threshold { + minutes = 15 + } + } } } } diff --git a/modules/storage/s3_bucket_2023/variables.tf b/modules/storage/s3_bucket_2023/variables.tf index 860152c..6906b16 100644 --- a/modules/storage/s3_bucket_2023/variables.tf +++ b/modules/storage/s3_bucket_2023/variables.tf @@ -49,3 +49,12 @@ variable replication_dest_bucket_name { default = null } +variable replication_destination_aws_account_id { + type = number + default = null +} + +variable replication_destination_kms_key_arn { + type = string + default = null +} \ No newline at end of file