NEW: s3 replication module
This commit is contained in:
parent
ae4d2477a9
commit
a7c781b7c0
GPG Key ID:
CD4FF6793F09AB86
50
modules/storage/s3-bucket-replication/README.md
Normal file
50
modules/storage/s3-bucket-replication/README.md
Normal file
@ -0,0 +1,50 @@
|
||||
<!-- This readme file is generated with terraform-docs -->
|
||||
## Requirements
|
||||
|
||||
| Name | Version |
|
||||
|------|---------|
|
||||
| terraform | >= 1.3.0 |
|
||||
| aws | >= 5.0 |
|
||||
|
||||
## Providers
|
||||
|
||||
| Name | Version |
|
||||
|------|---------|
|
||||
| aws | >= 5.0 |
|
||||
| random | n/a |
|
||||
|
||||
## Modules
|
||||
|
||||
No modules.
|
||||
|
||||
## Resources
|
||||
|
||||
| Name | Type |
|
||||
|------|------|
|
||||
| [aws_iam_role.replication-role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
|
||||
| [aws_iam_role_policy.role-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
|
||||
| [aws_s3_bucket_replication_configuration.replication-config](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_replication_configuration) | resource |
|
||||
| [random_id.rid](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource |
|
||||
| [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
|
||||
| [aws_iam_policy_document.replication-role-policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
|
||||
| [aws_s3_bucket.destination-bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/s3_bucket) | data source |
|
||||
| [aws_s3_bucket.source-bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/s3_bucket) | data source |
|
||||
|
||||
## Inputs
|
||||
|
||||
| Name | Description | Type | Default | Required |
|
||||
|------|-------------|------|------------------|:--------:|
|
||||
| destination-bucket-account-id | Account id of destination bucket. | `string` | `"111122223333"` | no |
|
||||
| destination-bucket-encryption-key-arn | Encryption key arn for destination bucket | `string` | n/a | yes |
|
||||
| destination-bucket-name | Name of destination bucket | `string` | n/a | yes |
|
||||
| source-bucket-name | Name of source s3 bucket | `string` | n/a | yes |
|
||||
|
||||
## Outputs
|
||||
|
||||
| Name | Description |
|
||||
|------|-------------|
|
||||
| replication-role-arn | n/a |
|
||||
|
||||
---
|
||||
## Authorship
|
||||
This module was developed by xpk.
|
||||
131
modules/storage/s3-bucket-replication/main.tf
Normal file
131
modules/storage/s3-bucket-replication/main.tf
Normal file
@ -0,0 +1,131 @@
|
||||
# sets up data sources for s3 buckets
|
||||
|
||||
data "aws_s3_bucket" "source-bucket" {
|
||||
bucket = var.source-bucket-name
|
||||
}
|
||||
|
||||
data "aws_s3_bucket" "destination-bucket" {
|
||||
bucket = var.destination-bucket-name
|
||||
}
|
||||
|
||||
# Create replication role in source account
|
||||
data "aws_iam_policy_document" "assume_role_policy" {
|
||||
statement {
|
||||
actions = ["sts:AssumeRole"]
|
||||
|
||||
principals {
|
||||
type = "Service"
|
||||
identifiers = ["s3.amazonaws.com"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "replication-role-policy" {
|
||||
statement {
|
||||
sid = "AccessToReplicaBucket"
|
||||
actions = [
|
||||
"s3:ReplicateObject",
|
||||
"s3:ReplicateDelete",
|
||||
"s3:ReplicateTags",
|
||||
"s3:ObjectOwnerOverrideToBucketOwner"
|
||||
]
|
||||
effect = "Allow"
|
||||
resources = [
|
||||
data.aws_s3_bucket.source-bucket.arn,
|
||||
data.aws_s3_bucket.destination-bucket.arn,
|
||||
"${data.aws_s3_bucket.source-bucket.arn}/*",
|
||||
"${data.aws_s3_bucket.destination-bucket.arn}/*"
|
||||
]
|
||||
}
|
||||
statement {
|
||||
sid = "ReadAccessOnSourceBuckets"
|
||||
actions = ["s3:Get*", "s3:List*"]
|
||||
effect = "Allow"
|
||||
resources = [
|
||||
data.aws_s3_bucket.source-bucket.arn,
|
||||
]
|
||||
}
|
||||
statement {
|
||||
sid = "ObjectAccessOnSourceBuckets"
|
||||
actions = [
|
||||
"s3:GetObjectVersionForReplication",
|
||||
"s3:GetObjectVersionAcl",
|
||||
"s3:GetObjectVersionTagging"
|
||||
]
|
||||
effect = "Allow"
|
||||
resources = [
|
||||
"${data.aws_s3_bucket.source-bucket.arn}/*"
|
||||
]
|
||||
}
|
||||
statement {
|
||||
sid = "DecryptSourceBucketObjects"
|
||||
actions = [
|
||||
"kms:Decrypt"
|
||||
]
|
||||
effect = "Allow"
|
||||
resources = ["*"]
|
||||
}
|
||||
statement {
|
||||
sid = "EncryptReplicaObjects"
|
||||
actions = [
|
||||
"kms:Encrypt"
|
||||
]
|
||||
effect = "Allow"
|
||||
resources = ["*"]
|
||||
}
|
||||
}
|
||||
|
||||
resource "random_id" "rid" {
|
||||
byte_length = 4
|
||||
}
|
||||
|
||||
resource "aws_iam_role" "replication-role" {
|
||||
name = "BucketReplicationRole${random_id.rid.dec}"
|
||||
assume_role_policy = data.aws_iam_policy_document.assume_role_policy.json
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy" "role-policy" {
|
||||
name = "bucket-replication"
|
||||
role = aws_iam_role.replication-role.id
|
||||
policy = data.aws_iam_policy_document.replication-role-policy.json
|
||||
}
|
||||
|
||||
# Setup bucket replication
|
||||
resource "aws_s3_bucket_replication_configuration" "replication-config" {
|
||||
role = aws_iam_role.replication-role.arn
|
||||
bucket = var.source-bucket-name
|
||||
|
||||
rule {
|
||||
id = "ReplicateAll"
|
||||
|
||||
status = "Enabled"
|
||||
|
||||
source_selection_criteria {
|
||||
sse_kms_encrypted_objects {
|
||||
status = "Enabled"
|
||||
}
|
||||
}
|
||||
|
||||
# V2 replication configurations
|
||||
delete_marker_replication {
|
||||
status = "Enabled"
|
||||
}
|
||||
|
||||
filter {
|
||||
}
|
||||
|
||||
destination {
|
||||
bucket = data.aws_s3_bucket.destination-bucket.arn
|
||||
account = var.destination-bucket-account-id
|
||||
access_control_translation {
|
||||
owner = "Destination"
|
||||
}
|
||||
encryption_configuration {
|
||||
replica_kms_key_id = var.destination-bucket-encryption-key-arn
|
||||
}
|
||||
metrics {
|
||||
status = "Enabled"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
3
modules/storage/s3-bucket-replication/outputs.tf
Normal file
3
modules/storage/s3-bucket-replication/outputs.tf
Normal file
@ -0,0 +1,3 @@
|
||||
output replication-role-arn {
|
||||
value = aws_iam_role.replication-role.arn
|
||||
}
|
||||
20
modules/storage/s3-bucket-replication/variables.tf
Normal file
20
modules/storage/s3-bucket-replication/variables.tf
Normal file
@ -0,0 +1,20 @@
|
||||
variable source-bucket-name {
|
||||
type = string
|
||||
description = "Name of source s3 bucket"
|
||||
}
|
||||
|
||||
variable destination-bucket-name {
|
||||
type = string
|
||||
description = "Name of destination bucket"
|
||||
}
|
||||
|
||||
variable destination-bucket-account-id {
|
||||
type = string
|
||||
description = "Account id of destination bucket. Defaults to BEA-SYS-LOG-UAT"
|
||||
default = "894849410890"
|
||||
}
|
||||
|
||||
variable destination-bucket-encryption-key-arn {
|
||||
type = string
|
||||
description = "Encryption key arn for destination bucket"
|
||||
}
|
||||
10
modules/storage/s3-bucket-replication/versions.tf
Normal file
10
modules/storage/s3-bucket-replication/versions.tf
Normal file
@ -0,0 +1,10 @@
|
||||
terraform {
|
||||
required_version = ">= 1.3.0"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = ">= 5.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -37,3 +37,227 @@ module "bucket1" {
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
## Note on bucket replication
|
||||
To securely replicate a bucket to a bucket in another aws account, kms key is required.
|
||||
|
||||
Steps to setup replication are:
|
||||
1. Create replication iam role on the source account, with an assume role policy trusting s3
|
||||
```json
|
||||
{
|
||||
"Effect":"Allow",
|
||||
"Principal":{
|
||||
"Service":"s3.amazonaws.com"
|
||||
},
|
||||
"Action":"sts:AssumeRole"
|
||||
}
|
||||
```
|
||||
The role needs permissions granted in the role iam policy. For example:
|
||||
```json
|
||||
{
|
||||
"Statement": [
|
||||
{
|
||||
"Action": [
|
||||
"s3:ListBucket",
|
||||
"s3:GetReplicationConfiguration"
|
||||
],
|
||||
"Effect": "Allow",
|
||||
"Resource": "arn:aws:s3:::whk1-bea-icc-mbk-prd-vpc01-flowlog-s3-accept",
|
||||
"Sid": ""
|
||||
},
|
||||
{
|
||||
"Action": [
|
||||
"s3:GetObjectVersionTagging",
|
||||
"s3:GetObjectVersionForReplication",
|
||||
"s3:GetObjectVersionAcl"
|
||||
],
|
||||
"Effect": "Allow",
|
||||
"Resource": "arn:aws:s3:::whk1-bea-icc-mbk-prd-vpc01-flowlog-s3-accept/*",
|
||||
"Sid": ""
|
||||
},
|
||||
{
|
||||
"Action": [
|
||||
"s3:ReplicateTags",
|
||||
"s3:ReplicateObject",
|
||||
"s3:ReplicateDelete",
|
||||
"s3:ObjectOwnerOverrideToBucketOwner"
|
||||
],
|
||||
"Effect": "Allow",
|
||||
"Resource": "arn:aws:s3:::whk1-bea-icc-log-mbk-prd-vpc01-flowlog-s3-accept/*",
|
||||
"Sid": ""
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"kms:Decrypt",
|
||||
"kms:GenerateDataKey"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:kms:ap-east-1:851239346925:key/708b6ece-05f5-40ed-a91c-dbcf2af46407"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"kms:GenerateDataKey",
|
||||
"kms:Encrypt"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:kms:ap-east-1:894849410890:key/b555d9d6-d451-4ec8-8ca2-cb6849cadee4"
|
||||
]
|
||||
}
|
||||
],
|
||||
"Version": "2012-10-17"
|
||||
}
|
||||
```
|
||||
If bucket key is used, then additional permission needs to be granted
|
||||
```json
|
||||
{
|
||||
"Action":[
|
||||
"kms:Decrypt"
|
||||
],
|
||||
"Effect":"Allow",
|
||||
"Condition":{
|
||||
"StringLike":{
|
||||
"kms:ViaService":"s3.ap-east-1.amazonaws.com",
|
||||
"kms:EncryptionContext:aws:s3:arn":[
|
||||
"arn:aws:s3:::<source-bucket-name>/*"
|
||||
]
|
||||
}
|
||||
},
|
||||
"Resource":[
|
||||
"arn:aws:kms:ap-east-1:<source-account-id>:key/<source-account-key-id>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Action":[
|
||||
"kms:Encrypt"
|
||||
],
|
||||
"Effect":"Allow",
|
||||
"Condition":{
|
||||
"StringLike":{
|
||||
"kms:ViaService":"s3.ap-east-1.amazonaws.com",
|
||||
"kms:EncryptionContext:aws:s3:arn":[
|
||||
"arn:aws:s3:::<dest-bucket-name>/*"
|
||||
]
|
||||
}
|
||||
},
|
||||
"Resource":[
|
||||
"arn:aws:kms:ap-east-1:<dest-account-id>:key/<dest-account-key-id>"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
2. On the destination account, grant access in KMS key policy
|
||||
```json
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Id": "key-consolepolicy-3",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "Enable IAM User Permissions",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"AWS": "arn:aws:iam::<dest-account-id>:root"
|
||||
},
|
||||
"Action": "kms:*",
|
||||
"Resource": "*"
|
||||
},
|
||||
{
|
||||
"Sid": "Allow use of the key",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"AWS": [
|
||||
"arn:aws:iam::<src-account-id>:root",
|
||||
"arn:aws:iam::<dest-account-id>:root"
|
||||
]
|
||||
},
|
||||
"Action": [
|
||||
"kms:Encrypt",
|
||||
"kms:Decrypt",
|
||||
"kms:ReEncrypt*",
|
||||
"kms:GenerateDataKey*",
|
||||
"kms:DescribeKey"
|
||||
],
|
||||
"Resource": "*"
|
||||
},
|
||||
{
|
||||
"Sid": "Allow attachment of persistent resources",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"AWS": [
|
||||
"arn:aws:iam::<src-account-id>:root",
|
||||
"arn:aws:iam::<dest-account-id>:root"
|
||||
]
|
||||
},
|
||||
"Action": [
|
||||
"kms:CreateGrant",
|
||||
"kms:ListGrants",
|
||||
"kms:RevokeGrant"
|
||||
],
|
||||
"Resource": "*",
|
||||
"Condition": {
|
||||
"Bool": {
|
||||
"kms:GrantIsForAWSResource": "true"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Sid": "Allow AWS Service to use the key",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"Service": [
|
||||
"s3.amazonaws.com",
|
||||
"delivery.logs.amazonaws.com",
|
||||
"cloudtrail.amazonaws.com"
|
||||
]
|
||||
},
|
||||
"Action": [
|
||||
"kms:Encrypt",
|
||||
"kms:Decrypt",
|
||||
"kms:ReEncrypt*",
|
||||
"kms:GenerateDataKey*",
|
||||
"kms:DescribeKey"
|
||||
],
|
||||
"Resource": "*"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
3. Edit destination bucket policy
|
||||
```json
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Id": "",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "Set permissions for objects",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"AWS": "arn:aws:iam::<src-account-id>:root"
|
||||
},
|
||||
"Action": [
|
||||
"s3:ReplicateDelete",
|
||||
"s3:ReplicateObject",
|
||||
"s3:ReplicateTags",
|
||||
"s3:ObjectOwnerOverrideToBucketOwner"
|
||||
],
|
||||
"Resource": "arn:aws:s3:::<dest-bucket-name>/*"
|
||||
},
|
||||
{
|
||||
"Sid": "Set permissions on bucket",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"AWS": "arn:aws:iam::<src-account-id>:root"
|
||||
},
|
||||
"Action": [
|
||||
"s3:List*",
|
||||
"s3:GetBucketVersioning",
|
||||
"s3:PutBucketVersioning"
|
||||
],
|
||||
"Resource": "arn:aws:s3:::<dest-bucket-name>"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
@ -11,9 +11,34 @@ resource "aws_s3_bucket_public_access_block" "block_public_access" {
|
||||
restrict_public_buckets = true
|
||||
}
|
||||
|
||||
# Add SecureTransport restriction by default
|
||||
data "aws_iam_policy_document" "bucket_policy" {
|
||||
source_policy_documents = [var.bucket_policy_json]
|
||||
|
||||
statement {
|
||||
sid = "AllowSSLRequestsOnly"
|
||||
actions = ["s3:*"]
|
||||
effect = "Deny"
|
||||
principals {
|
||||
type = "*"
|
||||
identifiers = ["*"]
|
||||
}
|
||||
resources = [
|
||||
aws_s3_bucket.this.arn,
|
||||
"${aws_s3_bucket.this.arn}/*"
|
||||
]
|
||||
condition {
|
||||
test = "Bool"
|
||||
values = [false]
|
||||
variable = "aws:SecureTransport"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_policy" "bucket_policy" {
|
||||
bucket = aws_s3_bucket.this.id
|
||||
policy = var.bucket_policy_json
|
||||
# policy = var.bucket_policy_json
|
||||
policy = data.aws_iam_policy_document.bucket_policy.json
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_lifecycle_configuration" "lifecycle" {
|
||||
@ -93,15 +118,54 @@ resource "aws_s3_bucket_versioning" "versioning" {
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_replication_configuration" "replication" {
|
||||
count = var.enable_replication && var.enable_versioning ? 1 : 0
|
||||
role = var.replication_role_arn
|
||||
bucket = aws_s3_bucket.this.id
|
||||
count = var.enable_replication && var.enable_versioning ? 1 : 0
|
||||
role = var.replication_role_arn
|
||||
bucket = aws_s3_bucket.this.id
|
||||
|
||||
|
||||
rule {
|
||||
id = "replrule1"
|
||||
status = "Enabled"
|
||||
delete_marker_replication {
|
||||
status = "Enabled"
|
||||
}
|
||||
|
||||
source_selection_criteria {
|
||||
replica_modifications {
|
||||
status = "Enabled"
|
||||
}
|
||||
sse_kms_encrypted_objects {
|
||||
status = "Enabled"
|
||||
}
|
||||
}
|
||||
|
||||
destination {
|
||||
bucket = var.replication_dest_bucket_name
|
||||
storage_class = "INTELLIGENT_TIERING"
|
||||
|
||||
account = var.replication_destination_aws_account_id
|
||||
|
||||
encryption_configuration {
|
||||
replica_kms_key_id = var.replication_destination_kms_key_arn
|
||||
}
|
||||
|
||||
access_control_translation {
|
||||
owner = "Destination"
|
||||
}
|
||||
|
||||
replication_time {
|
||||
status = "Enabled"
|
||||
time {
|
||||
minutes = 15
|
||||
}
|
||||
}
|
||||
|
||||
metrics {
|
||||
status = "Enabled"
|
||||
event_threshold {
|
||||
minutes = 15
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -49,3 +49,12 @@ variable replication_dest_bucket_name {
|
||||
default = null
|
||||
}
|
||||
|
||||
variable replication_destination_aws_account_id {
|
||||
type = number
|
||||
default = null
|
||||
}
|
||||
|
||||
variable replication_destination_kms_key_arn {
|
||||
type = string
|
||||
default = null
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user