NEW: s3 replication module

This commit is contained in:
xpk 2024-01-12 13:49:50 +08:00
parent ae4d2477a9
commit a7c781b7c0
Signed by: xpk
GPG Key ID: CD4FF6793F09AB86
8 changed files with 516 additions and 5 deletions

View File

@ -0,0 +1,50 @@
<!-- This readme file is generated with terraform-docs -->
## Requirements
| Name | Version |
|------|---------|
| terraform | >= 1.3.0 |
| aws | >= 5.0 |
## Providers
| Name | Version |
|------|---------|
| aws | >= 5.0 |
| random | n/a |
## Modules
No modules.
## Resources
| Name | Type |
|------|------|
| [aws_iam_role.replication-role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
| [aws_iam_role_policy.role-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
| [aws_s3_bucket_replication_configuration.replication-config](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_replication_configuration) | resource |
| [random_id.rid](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource |
| [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
| [aws_iam_policy_document.replication-role-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
| [aws_s3_bucket.destination-bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/s3_bucket) | data source |
| [aws_s3_bucket.source-bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/s3_bucket) | data source |
## Inputs
| Name | Description | Type | Default | Required |
|------|-------------|------|------------------|:--------:|
| destination-bucket-account-id | Account id of destination bucket. | `string` | `"111122223333"` | no |
| destination-bucket-encryption-key-arn | Encryption key arn for destination bucket | `string` | n/a | yes |
| destination-bucket-name | Name of destination bucket | `string` | n/a | yes |
| source-bucket-name | Name of source s3 bucket | `string` | n/a | yes |
## Outputs
| Name | Description |
|------|-------------|
| replication-role-arn | n/a |
---
## Authorship
This module was developed by xpk.

View File

@ -0,0 +1,131 @@
# sets up data sources for s3 buckets
data "aws_s3_bucket" "source-bucket" {
bucket = var.source-bucket-name
}
data "aws_s3_bucket" "destination-bucket" {
bucket = var.destination-bucket-name
}
# Create replication role in source account
data "aws_iam_policy_document" "assume_role_policy" {
statement {
actions = ["sts:AssumeRole"]
principals {
type = "Service"
identifiers = ["s3.amazonaws.com"]
}
}
}
data "aws_iam_policy_document" "replication-role-policy" {
statement {
sid = "AccessToReplicaBucket"
actions = [
"s3:ReplicateObject",
"s3:ReplicateDelete",
"s3:ReplicateTags",
"s3:ObjectOwnerOverrideToBucketOwner"
]
effect = "Allow"
resources = [
data.aws_s3_bucket.source-bucket.arn,
data.aws_s3_bucket.destination-bucket.arn,
"${data.aws_s3_bucket.source-bucket.arn}/*",
"${data.aws_s3_bucket.destination-bucket.arn}/*"
]
}
statement {
sid = "ReadAccessOnSourceBuckets"
actions = ["s3:Get*", "s3:List*"]
effect = "Allow"
resources = [
data.aws_s3_bucket.source-bucket.arn,
]
}
statement {
sid = "ObjectAccessOnSourceBuckets"
actions = [
"s3:GetObjectVersionForReplication",
"s3:GetObjectVersionAcl",
"s3:GetObjectVersionTagging"
]
effect = "Allow"
resources = [
"${data.aws_s3_bucket.source-bucket.arn}/*"
]
}
statement {
sid = "DecryptSourceBucketObjects"
actions = [
"kms:Decrypt"
]
effect = "Allow"
resources = ["*"]
}
statement {
sid = "EncryptReplicaObjects"
actions = [
"kms:Encrypt"
]
effect = "Allow"
resources = ["*"]
}
}
resource "random_id" "rid" {
byte_length = 4
}
resource "aws_iam_role" "replication-role" {
name = "BucketReplicationRole${random_id.rid.dec}"
assume_role_policy = data.aws_iam_policy_document.assume_role_policy.json
}
resource "aws_iam_role_policy" "role-policy" {
name = "bucket-replication"
role = aws_iam_role.replication-role.id
policy = data.aws_iam_policy_document.replication-role-policy.json
}
# Setup bucket replication
resource "aws_s3_bucket_replication_configuration" "replication-config" {
role = aws_iam_role.replication-role.arn
bucket = var.source-bucket-name
rule {
id = "ReplicateAll"
status = "Enabled"
source_selection_criteria {
sse_kms_encrypted_objects {
status = "Enabled"
}
}
# V2 replication configurations
delete_marker_replication {
status = "Enabled"
}
filter {
}
destination {
bucket = data.aws_s3_bucket.destination-bucket.arn
account = var.destination-bucket-account-id
access_control_translation {
owner = "Destination"
}
encryption_configuration {
replica_kms_key_id = var.destination-bucket-encryption-key-arn
}
metrics {
status = "Enabled"
}
}
}
}

View File

@ -0,0 +1,3 @@
output replication-role-arn {
value = aws_iam_role.replication-role.arn
}

View File

@ -0,0 +1,20 @@
variable source-bucket-name {
type = string
description = "Name of source s3 bucket"
}
variable destination-bucket-name {
type = string
description = "Name of destination bucket"
}
variable destination-bucket-account-id {
type = string
description = "Account id of destination bucket. Defaults to BEA-SYS-LOG-UAT"
default = "894849410890"
}
variable destination-bucket-encryption-key-arn {
type = string
description = "Encryption key arn for destination bucket"
}

View File

@ -0,0 +1,10 @@
terraform {
required_version = ">= 1.3.0"
required_providers {
aws = {
source = "hashicorp/aws"
version = ">= 5.0"
}
}
}

View File

@ -37,3 +37,227 @@ module "bucket1" {
} }
``` ```
## Note on bucket replication
To securely replicate a bucket to a bucket in another aws account, kms key is required.
Steps to setup replication are:
1. Create replication iam role on the source account, with an assume role policy trusting s3
```json
{
"Effect":"Allow",
"Principal":{
"Service":"s3.amazonaws.com"
},
"Action":"sts:AssumeRole"
}
```
The role needs permissions granted in the role iam policy. For example:
```json
{
"Statement": [
{
"Action": [
"s3:ListBucket",
"s3:GetReplicationConfiguration"
],
"Effect": "Allow",
"Resource": "arn:aws:s3:::whk1-bea-icc-mbk-prd-vpc01-flowlog-s3-accept",
"Sid": ""
},
{
"Action": [
"s3:GetObjectVersionTagging",
"s3:GetObjectVersionForReplication",
"s3:GetObjectVersionAcl"
],
"Effect": "Allow",
"Resource": "arn:aws:s3:::whk1-bea-icc-mbk-prd-vpc01-flowlog-s3-accept/*",
"Sid": ""
},
{
"Action": [
"s3:ReplicateTags",
"s3:ReplicateObject",
"s3:ReplicateDelete",
"s3:ObjectOwnerOverrideToBucketOwner"
],
"Effect": "Allow",
"Resource": "arn:aws:s3:::whk1-bea-icc-log-mbk-prd-vpc01-flowlog-s3-accept/*",
"Sid": ""
},
{
"Effect": "Allow",
"Action": [
"kms:Decrypt",
"kms:GenerateDataKey"
],
"Resource": [
"arn:aws:kms:ap-east-1:851239346925:key/708b6ece-05f5-40ed-a91c-dbcf2af46407"
]
},
{
"Effect": "Allow",
"Action": [
"kms:GenerateDataKey",
"kms:Encrypt"
],
"Resource": [
"arn:aws:kms:ap-east-1:894849410890:key/b555d9d6-d451-4ec8-8ca2-cb6849cadee4"
]
}
],
"Version": "2012-10-17"
}
```
If bucket key is used, then additional permission needs to be granted
```json
{
"Action":[
"kms:Decrypt"
],
"Effect":"Allow",
"Condition":{
"StringLike":{
"kms:ViaService":"s3.ap-east-1.amazonaws.com",
"kms:EncryptionContext:aws:s3:arn":[
"arn:aws:s3:::<source-bucket-name>/*"
]
}
},
"Resource":[
"arn:aws:kms:ap-east-1:<source-account-id>:key/<source-account-key-id>"
]
},
{
"Action":[
"kms:Encrypt"
],
"Effect":"Allow",
"Condition":{
"StringLike":{
"kms:ViaService":"s3.ap-east-1.amazonaws.com",
"kms:EncryptionContext:aws:s3:arn":[
"arn:aws:s3:::<dest-bucket-name>/*"
]
}
},
"Resource":[
"arn:aws:kms:ap-east-1:<dest-account-id>:key/<dest-account-key-id>"
]
}
```
2. On the destination account, grant access in KMS key policy
```json
{
"Version": "2012-10-17",
"Id": "key-consolepolicy-3",
"Statement": [
{
"Sid": "Enable IAM User Permissions",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::<dest-account-id>:root"
},
"Action": "kms:*",
"Resource": "*"
},
{
"Sid": "Allow use of the key",
"Effect": "Allow",
"Principal": {
"AWS": [
"arn:aws:iam::<src-account-id>:root",
"arn:aws:iam::<dest-account-id>:root"
]
},
"Action": [
"kms:Encrypt",
"kms:Decrypt",
"kms:ReEncrypt*",
"kms:GenerateDataKey*",
"kms:DescribeKey"
],
"Resource": "*"
},
{
"Sid": "Allow attachment of persistent resources",
"Effect": "Allow",
"Principal": {
"AWS": [
"arn:aws:iam::<src-account-id>:root",
"arn:aws:iam::<dest-account-id>:root"
]
},
"Action": [
"kms:CreateGrant",
"kms:ListGrants",
"kms:RevokeGrant"
],
"Resource": "*",
"Condition": {
"Bool": {
"kms:GrantIsForAWSResource": "true"
}
}
},
{
"Sid": "Allow AWS Service to use the key",
"Effect": "Allow",
"Principal": {
"Service": [
"s3.amazonaws.com",
"delivery.logs.amazonaws.com",
"cloudtrail.amazonaws.com"
]
},
"Action": [
"kms:Encrypt",
"kms:Decrypt",
"kms:ReEncrypt*",
"kms:GenerateDataKey*",
"kms:DescribeKey"
],
"Resource": "*"
}
]
}
```
3. Edit destination bucket policy
```json
{
"Version": "2012-10-17",
"Id": "",
"Statement": [
{
"Sid": "Set permissions for objects",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::<src-account-id>:root"
},
"Action": [
"s3:ReplicateDelete",
"s3:ReplicateObject",
"s3:ReplicateTags",
"s3:ObjectOwnerOverrideToBucketOwner"
],
"Resource": "arn:aws:s3:::<dest-bucket-name>/*"
},
{
"Sid": "Set permissions on bucket",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::<src-account-id>:root"
},
"Action": [
"s3:List*",
"s3:GetBucketVersioning",
"s3:PutBucketVersioning"
],
"Resource": "arn:aws:s3:::<dest-bucket-name>"
}
]
}
```

View File

@ -11,9 +11,34 @@ resource "aws_s3_bucket_public_access_block" "block_public_access" {
restrict_public_buckets = true restrict_public_buckets = true
} }
# Add SecureTransport restriction by default
data "aws_iam_policy_document" "bucket_policy" {
source_policy_documents = [var.bucket_policy_json]
statement {
sid = "AllowSSLRequestsOnly"
actions = ["s3:*"]
effect = "Deny"
principals {
type = "*"
identifiers = ["*"]
}
resources = [
aws_s3_bucket.this.arn,
"${aws_s3_bucket.this.arn}/*"
]
condition {
test = "Bool"
values = [false]
variable = "aws:SecureTransport"
}
}
}
resource "aws_s3_bucket_policy" "bucket_policy" { resource "aws_s3_bucket_policy" "bucket_policy" {
bucket = aws_s3_bucket.this.id bucket = aws_s3_bucket.this.id
policy = var.bucket_policy_json # policy = var.bucket_policy_json
policy = data.aws_iam_policy_document.bucket_policy.json
} }
resource "aws_s3_bucket_lifecycle_configuration" "lifecycle" { resource "aws_s3_bucket_lifecycle_configuration" "lifecycle" {
@ -96,12 +121,51 @@ resource "aws_s3_bucket_replication_configuration" "replication" {
count = var.enable_replication && var.enable_versioning ? 1 : 0 count = var.enable_replication && var.enable_versioning ? 1 : 0
role = var.replication_role_arn role = var.replication_role_arn
bucket = aws_s3_bucket.this.id bucket = aws_s3_bucket.this.id
rule { rule {
id = "replrule1" id = "replrule1"
status = "Enabled" status = "Enabled"
delete_marker_replication {
status = "Enabled"
}
source_selection_criteria {
replica_modifications {
status = "Enabled"
}
sse_kms_encrypted_objects {
status = "Enabled"
}
}
destination { destination {
bucket = var.replication_dest_bucket_name bucket = var.replication_dest_bucket_name
storage_class = "INTELLIGENT_TIERING" storage_class = "INTELLIGENT_TIERING"
account = var.replication_destination_aws_account_id
encryption_configuration {
replica_kms_key_id = var.replication_destination_kms_key_arn
}
access_control_translation {
owner = "Destination"
}
replication_time {
status = "Enabled"
time {
minutes = 15
}
}
metrics {
status = "Enabled"
event_threshold {
minutes = 15
}
}
} }
} }
} }

View File

@ -49,3 +49,12 @@ variable replication_dest_bucket_name {
default = null default = null
} }
variable replication_destination_aws_account_id {
type = number
default = null
}
variable replication_destination_kms_key_arn {
type = string
default = null
}