From 6584960d1a25102ef5ef4571158c9137e3ad527e Mon Sep 17 00:00:00 2001 From: xpk Date: Tue, 23 May 2023 13:10:16 +0800 Subject: [PATCH] UPD: backported monitoring modules from customer repo --- .../Monitoring.ALB/main.tf | 107 ++++++++-- .../Monitoring.ALB/outputs.tf | 4 + .../Monitoring.ALB/provider.tf | 2 +- .../Monitoring.ALB/variables.tf | 6 +- .../Monitoring.ASG/main.tf | 31 ++- .../Monitoring.ASG/provider.tf | 2 +- .../Monitoring.ASG/variables.tf | 1 + .../Monitoring.EC2/get-cwagent-device.sh | 3 +- .../Monitoring.EC2/get-cwagent-dimensions.sh | 3 +- .../Monitoring.EC2/get-os-platform.sh | 3 +- .../Monitoring.EC2/main.tf | 185 ++++-------------- .../Monitoring.EC2/provider.tf | 2 +- .../Monitoring.EC2/variables.tf | 2 +- .../Monitoring.EKS/main.tf | 15 +- .../Monitoring.EKS/provider.tf | 2 +- .../Monitoring.EMR/main.tf | 32 +-- .../Monitoring.EMR/provider.tf | 2 +- .../Monitoring.EventBridge/main.tf | 2 +- .../Monitoring.EventBridge/provider.tf | 2 +- .../Monitoring.Kafka/main.tf | 33 +--- .../Monitoring.Kafka/provider.tf | 2 +- .../Monitoring.NGW/main.tf | 77 +------- .../Monitoring.NGW/provider.tf | 2 +- .../Monitoring.NLB/main.tf | 43 +++- .../Monitoring.NLB/outputs.tf | 4 + .../Monitoring.NLB/provider.tf | 2 +- .../Monitoring.NLB/variables.tf | 3 +- .../Monitoring.OpenSearch/main.tf | 5 +- .../Monitoring.OpenSearch/provider.tf | 2 +- .../Monitoring.RDS/README.md | 1 + .../Monitoring.RDS/main.tf | 5 +- .../Monitoring.RDS/provider.tf | 2 +- .../Monitoring.Redis/main.tf | 7 +- .../Monitoring.Redis/provider.tf | 2 +- .../Monitoring.TGW/main.tf | 24 ++- .../Monitoring.TGW/provider.tf | 2 +- modules/util/resource-list/README.md | 2 + .../resource-list/list-alb-targetgroups.sh | 6 + modules/util/resource-list/list-emr.sh | 3 + modules/util/resource-list/list-kafka.sh | 4 +- .../resource-list/list-nlb-targetgroups.sh | 3 +- modules/util/resource-list/list-opensearch.sh | 4 +- modules/util/resource-list/list-redis.sh | 3 + modules/util/resource-list/list-tgw.sh | 4 + modules/util/resource-list/main.tf | 2 +- modules/util/resource-list/variables.tf | 9 + modules/util/terraform-aws-cli/.gitignore | 5 + .../terraform-aws-cli/.pre-commit-config.yaml | 43 ++++ .../util/terraform-aws-cli/.terraform-version | 1 + modules/util/terraform-aws-cli/.tflint.hcl | 32 +++ modules/util/terraform-aws-cli/.travis.yml | 12 ++ modules/util/terraform-aws-cli/CHANGELOG.md | 86 ++++++++ modules/util/terraform-aws-cli/README.md | 131 +++++++++++++ modules/util/terraform-aws-cli/main.tf | 42 ++++ .../scripts/awsWithAssumeRole.sh | 65 ++++++ .../tests/bad_arn/terraform.tfvars | 3 + .../terraform-aws-cli/tests/bad_arn/test.sh | 20 ++ .../util/terraform-aws-cli/tests/common.sh | 30 +++ .../empty_result/expected_variables.json | 24 +++ .../tests/empty_result/notes.txt | 26 +++ .../tests/empty_result/terraform.tfvars | 3 + .../tests/empty_result/test.sh | 41 ++++ .../terraform.tfvars | 4 + .../test.sh | 20 ++ .../expected_variables.json | 23 +++ .../terraform.tfvars | 3 + .../tests/role_session_name_optional/test.sh | 40 ++++ .../terraform.tfvars | 4 + .../tests/role_session_name_too_long/test.sh | 20 ++ .../test_with_debug/expected_variables.json | 23 +++ .../tests/test_with_debug/terraform.tfvars | 5 + .../tests/test_with_debug/test.sh | 40 ++++ .../expected_variables.json | 23 +++ .../tests/test_without_debug/terraform.tfvars | 4 + .../tests/test_without_debug/test.sh | 40 ++++ modules/util/terraform-aws-cli/tests/tests.sh | 4 + modules/util/terraform-aws-cli/variables.tf | 43 ++++ modules/util/terraform-aws-cli/versions.tf | 13 ++ 78 files changed, 1163 insertions(+), 372 deletions(-) create mode 100644 modules/ManagementGovernance/Monitoring.ALB/outputs.tf create mode 100644 modules/ManagementGovernance/Monitoring.NLB/outputs.tf create mode 100644 modules/util/resource-list/README.md create mode 100755 modules/util/resource-list/list-alb-targetgroups.sh create mode 100644 modules/util/terraform-aws-cli/.gitignore create mode 100644 modules/util/terraform-aws-cli/.pre-commit-config.yaml create mode 100644 modules/util/terraform-aws-cli/.terraform-version create mode 100644 modules/util/terraform-aws-cli/.tflint.hcl create mode 100644 modules/util/terraform-aws-cli/.travis.yml create mode 100644 modules/util/terraform-aws-cli/CHANGELOG.md create mode 100644 modules/util/terraform-aws-cli/README.md create mode 100644 modules/util/terraform-aws-cli/main.tf create mode 100755 modules/util/terraform-aws-cli/scripts/awsWithAssumeRole.sh create mode 100644 modules/util/terraform-aws-cli/tests/bad_arn/terraform.tfvars create mode 100755 modules/util/terraform-aws-cli/tests/bad_arn/test.sh create mode 100644 modules/util/terraform-aws-cli/tests/common.sh create mode 100644 modules/util/terraform-aws-cli/tests/empty_result/expected_variables.json create mode 100644 modules/util/terraform-aws-cli/tests/empty_result/notes.txt create mode 100644 modules/util/terraform-aws-cli/tests/empty_result/terraform.tfvars create mode 100755 modules/util/terraform-aws-cli/tests/empty_result/test.sh create mode 100644 modules/util/terraform-aws-cli/tests/role_session_name_invalid_characters/terraform.tfvars create mode 100755 modules/util/terraform-aws-cli/tests/role_session_name_invalid_characters/test.sh create mode 100644 modules/util/terraform-aws-cli/tests/role_session_name_optional/expected_variables.json create mode 100644 modules/util/terraform-aws-cli/tests/role_session_name_optional/terraform.tfvars create mode 100755 modules/util/terraform-aws-cli/tests/role_session_name_optional/test.sh create mode 100644 modules/util/terraform-aws-cli/tests/role_session_name_too_long/terraform.tfvars create mode 100755 modules/util/terraform-aws-cli/tests/role_session_name_too_long/test.sh create mode 100644 modules/util/terraform-aws-cli/tests/test_with_debug/expected_variables.json create mode 100644 modules/util/terraform-aws-cli/tests/test_with_debug/terraform.tfvars create mode 100755 modules/util/terraform-aws-cli/tests/test_with_debug/test.sh create mode 100644 modules/util/terraform-aws-cli/tests/test_without_debug/expected_variables.json create mode 100644 modules/util/terraform-aws-cli/tests/test_without_debug/terraform.tfvars create mode 100755 modules/util/terraform-aws-cli/tests/test_without_debug/test.sh create mode 100755 modules/util/terraform-aws-cli/tests/tests.sh create mode 100644 modules/util/terraform-aws-cli/variables.tf create mode 100644 modules/util/terraform-aws-cli/versions.tf diff --git a/modules/ManagementGovernance/Monitoring.ALB/main.tf b/modules/ManagementGovernance/Monitoring.ALB/main.tf index c64f5b9..95ff8a4 100644 --- a/modules/ManagementGovernance/Monitoring.ALB/main.tf +++ b/modules/ManagementGovernance/Monitoring.ALB/main.tf @@ -1,31 +1,104 @@ -data "external" "alb-targetgroups" { - program = ["bash", "../../modules/ManagementGovernance/Monitoring.ALB/list-alb-targetgroups.sh"] - query = { - lb = var.load-balancer +locals { + alb-name = "app/${split("/", var.load-balancer)[2]}/${split("/", var.load-balancer)[3]}" +} + +resource "aws_cloudwatch_metric_alarm" "alb-HTTPCode_ELB_5XX_Count" { + alarm_name = "${var.settings.HTTPCode_ELB_5XX_Count.ecccode}-ALB_${local.alb-name}-HTTPCode_ELB_5XX_Count" + comparison_operator = var.settings.HTTPCode_ELB_5XX_Count.comparison_operator + evaluation_periods = var.settings.HTTPCode_ELB_5XX_Count.evaluation_periods + metric_name = "HTTPCode_ELB_5XX_Count" + period = var.settings.HTTPCode_ELB_5XX_Count.period + statistic = var.settings.HTTPCode_ELB_5XX_Count.statistic + threshold = var.settings.HTTPCode_ELB_5XX_Count.threshold + alarm_description = "ALB:HTTPCode_ELB_5XX_Count" + namespace = "AWS/ApplicationELB" + insufficient_data_actions = [] + actions_enabled = var.actions-enabled + alarm_actions = [var.settings.HTTPCode_ELB_5XX_Count.action] + ok_actions = [var.settings.HTTPCode_ELB_5XX_Count.action] + dimensions = { + LoadBalancer = local.alb-name } + tags = var.default-tags +} + +resource "aws_cloudwatch_metric_alarm" "alb-TargetConnectionErrorCount" { + alarm_name = "${var.settings.TargetConnectionErrorCount.ecccode}-ALB_${local.alb-name}-TargetConnectionErrorCount" + comparison_operator = var.settings.TargetConnectionErrorCount.comparison_operator + evaluation_periods = var.settings.TargetConnectionErrorCount.evaluation_periods + metric_name = "TargetConnectionErrorCount" + period = var.settings.TargetConnectionErrorCount.period + statistic = var.settings.TargetConnectionErrorCount.statistic + threshold = var.settings.TargetConnectionErrorCount.threshold + alarm_description = "ALB:TargetConnectionErrorCount" + namespace = "AWS/ApplicationELB" + insufficient_data_actions = [] + actions_enabled = var.actions-enabled + alarm_actions = [var.settings.TargetConnectionErrorCount.action] + ok_actions = [var.settings.TargetConnectionErrorCount.action] + dimensions = { + LoadBalancer = local.alb-name + } + tags = var.default-tags +} + +resource "aws_cloudwatch_metric_alarm" "alb-TargetResponseTime" { + alarm_name = "${var.settings.TargetResponseTime.ecccode}-ALB_${local.alb-name}-TargetResponseTime" + comparison_operator = var.settings.TargetResponseTime.comparison_operator + evaluation_periods = var.settings.TargetResponseTime.evaluation_periods + metric_name = "TargetResponseTime" + period = var.settings.TargetResponseTime.period + statistic = var.settings.TargetResponseTime.statistic + threshold = var.settings.TargetResponseTime.threshold + alarm_description = "ALB:TargetResponseTime" + namespace = "AWS/ApplicationELB" + insufficient_data_actions = [] + actions_enabled = var.actions-enabled + alarm_actions = [var.settings.TargetResponseTime.action] + ok_actions = [var.settings.TargetResponseTime.action] + dimensions = { + LoadBalancer = local.alb-name + } + tags = var.default-tags +} + +/* +module "alb-targetgroups" { + source = "../../util/resource-list" + resource-type = "alb-targetgroups" + query-input = var.load-balancer + asrolearn = var.asrolearn +} +*/ +// causes Rate exceeded error, maybe because of adaptive AWS_RETRY_MODE? + +module "alb_tgs" { + assume_role_arn = var.asrolearn + role_session_name = "terraform-resource-list" + source = "../../util/terraform-aws-cli" + aws_cli_commands = ["elbv2", "describe-target-groups", "--load-balancer-arn", var.load-balancer] + aws_cli_query = "TargetGroups[*].TargetGroupArn" } resource "aws_cloudwatch_metric_alarm" "alb-HealthyHostCount" { - for_each = toset(split(" ", data.external.alb-targetgroups.result.result)) - alarm_name = "${var.cw-alarm-prefix}:ALBTG:HealthyHostCount:${split("/", each.value)[1]}/${split("/", each.value)[2]}" - comparison_operator = "LessThanThreshold" - evaluation_periods = "1" + # for_each = module.alb-targetgroups.result-set + for_each = toset(flatten(module.alb_tgs.result)) + alarm_name = "${var.settings.HealthHostCountMin.ecccode}-ALBTG_:${split(":", each.value)[5]}-HealthyHostCount" + comparison_operator = var.settings.HealthHostCountMin.comparison_operator + evaluation_periods = var.settings.HealthHostCountMin.evaluation_periods metric_name = "HealthyHostCount" - period = "300" - statistic = "Minimum" - threshold = var.threshold-HealthHostCountMin + period = var.settings.HealthHostCountMin.period + statistic = var.settings.HealthHostCountMin.statistic + threshold = var.settings.HealthHostCountMin.threshold alarm_description = "ALBTG:HealthyHostCount" namespace = "AWS/ApplicationELB" insufficient_data_actions = [] actions_enabled = var.actions-enabled - alarm_actions = [var.sns-targets.alarm-actions-emergency] - ok_actions = [var.sns-targets.alarm-actions-emergency] + alarm_actions = [var.settings.HealthHostCountMin.action] + ok_actions = [var.settings.HealthHostCountMin.action] dimensions = { - TargetGroup = "targetgroup/${split("/", each.value)[1]}/${split("/", each.value)[2]}" + TargetGroup = split(":", each.value)[5] LoadBalancer = "app/${split("/", var.load-balancer)[2]}/${split("/", var.load-balancer)[3]}" } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } diff --git a/modules/ManagementGovernance/Monitoring.ALB/outputs.tf b/modules/ManagementGovernance/Monitoring.ALB/outputs.tf new file mode 100644 index 0000000..b30ed9b --- /dev/null +++ b/modules/ManagementGovernance/Monitoring.ALB/outputs.tf @@ -0,0 +1,4 @@ +output alb-tg-count { + # value = length(module.alb-targetgroups.result-set) + value = length(flatten(module.alb_tgs.result)) +} \ No newline at end of file diff --git a/modules/ManagementGovernance/Monitoring.ALB/provider.tf b/modules/ManagementGovernance/Monitoring.ALB/provider.tf index 7b64cf5..933a1eb 100644 --- a/modules/ManagementGovernance/Monitoring.ALB/provider.tf +++ b/modules/ManagementGovernance/Monitoring.ALB/provider.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.36.1" + version = ">= 4.36.1" } } } diff --git a/modules/ManagementGovernance/Monitoring.ALB/variables.tf b/modules/ManagementGovernance/Monitoring.ALB/variables.tf index 35c1740..902f979 100644 --- a/modules/ManagementGovernance/Monitoring.ALB/variables.tf +++ b/modules/ManagementGovernance/Monitoring.ALB/variables.tf @@ -1,6 +1,6 @@ variable cw-alarm-prefix {} variable actions-enabled {} variable load-balancer {} -variable threshold-HealthHostCountMin {} -variable sns-targets {} -variable default-tags {} \ No newline at end of file +variable settings {} +variable default-tags {} +variable asrolearn {} \ No newline at end of file diff --git a/modules/ManagementGovernance/Monitoring.ASG/main.tf b/modules/ManagementGovernance/Monitoring.ASG/main.tf index 39d1a9a..f43d3d9 100644 --- a/modules/ManagementGovernance/Monitoring.ASG/main.tf +++ b/modules/ManagementGovernance/Monitoring.ASG/main.tf @@ -1,5 +1,9 @@ +data "aws_autoscaling_group" "asg" { + name = var.asg-name +} + resource "aws_cloudwatch_metric_alarm" "asg-CPUUtilization" { - alarm_name = "${var.cw-alarm-prefix}:ASG:CPUUtilization:${var.asg-name}" + alarm_name = "${var.settings.CPUUtilization.ecccode}-ASG_${var.asg-name}-CPUUtilization" comparison_operator = var.settings.CPUUtilization.comparison_operator evaluation_periods = var.settings.CPUUtilization.evaluation_periods metric_name = "CPUUtilization" @@ -12,11 +16,28 @@ resource "aws_cloudwatch_metric_alarm" "asg-CPUUtilization" { actions_enabled = var.actions-enabled alarm_actions = [var.settings.CPUUtilization.action] ok_actions = [var.settings.CPUUtilization.action] + dimensions = { + AutoScalingGroupName =var.asg-name + } + tags = var.default-tags +} + +resource "aws_cloudwatch_metric_alarm" "asg-GroupInServiceCapacity" { + alarm_name = "${var.settings.GroupInServiceCapacity.ecccode}-ASG_${var.asg-name}-GroupInServiceCapacity" + comparison_operator = "LessThanThreshold" + evaluation_periods = var.settings.GroupInServiceCapacity.evaluation_periods + metric_name = "GroupInServiceCapacity" + period = var.settings.GroupInServiceCapacity.period + statistic = "Minimum" + threshold = data.aws_autoscaling_group.asg.min_size + alarm_description = "ASG:GroupInServiceCapacity" + namespace = "AWS/AutoScaling" + insufficient_data_actions = [] + actions_enabled = var.actions-enabled + alarm_actions = [var.settings.GroupInServiceCapacity.action] + ok_actions = [var.settings.GroupInServiceCapacity.action] dimensions = { AutoScalingGroupName = var.asg-name } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } -} +} \ No newline at end of file diff --git a/modules/ManagementGovernance/Monitoring.ASG/provider.tf b/modules/ManagementGovernance/Monitoring.ASG/provider.tf index 7b64cf5..933a1eb 100644 --- a/modules/ManagementGovernance/Monitoring.ASG/provider.tf +++ b/modules/ManagementGovernance/Monitoring.ASG/provider.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.36.1" + version = ">= 4.36.1" } } } diff --git a/modules/ManagementGovernance/Monitoring.ASG/variables.tf b/modules/ManagementGovernance/Monitoring.ASG/variables.tf index 2f916c3..3d21e70 100644 --- a/modules/ManagementGovernance/Monitoring.ASG/variables.tf +++ b/modules/ManagementGovernance/Monitoring.ASG/variables.tf @@ -3,3 +3,4 @@ variable actions-enabled {} variable asg-name {} variable settings {} variable default-tags {} +variable ecccode {} \ No newline at end of file diff --git a/modules/ManagementGovernance/Monitoring.EC2/get-cwagent-device.sh b/modules/ManagementGovernance/Monitoring.EC2/get-cwagent-device.sh index 3fb6ec9..bf513e9 100755 --- a/modules/ManagementGovernance/Monitoring.EC2/get-cwagent-device.sh +++ b/modules/ManagementGovernance/Monitoring.EC2/get-cwagent-device.sh @@ -1,5 +1,6 @@ #!/bin/bash -eval "$(jq -r '@sh "id=\(.input)"')" +eval "$(jq -r '@sh "export id=\(.input) asrolearn=\(.asrolearn)"')" +eval $(aws sts assume-role --role-arn $asrolearn --role-session-name awscli | jq -cr '"export AWS_ACCESS_KEY_ID=" + .Credentials.AccessKeyId, "export AWS_SECRET_ACCESS_KEY=" + .Credentials.SecretAccessKey, "export AWS_SESSION_TOKEN=" + .Credentials.SessionToken, "export AWS_SESSION_EXPIRATION=" + .Credentials.Expiration') aws cloudwatch list-metrics --namespace CWAgent --metric-name disk_inodes_free \ --dimensions Name=InstanceId,Value=$id Name=path,Value=/ | \ diff --git a/modules/ManagementGovernance/Monitoring.EC2/get-cwagent-dimensions.sh b/modules/ManagementGovernance/Monitoring.EC2/get-cwagent-dimensions.sh index 16de537..b853235 100755 --- a/modules/ManagementGovernance/Monitoring.EC2/get-cwagent-dimensions.sh +++ b/modules/ManagementGovernance/Monitoring.EC2/get-cwagent-dimensions.sh @@ -1,5 +1,6 @@ #!/bin/bash -eval "$(jq -r '@sh "id=\(.input)"')" +eval "$(jq -r '@sh "export id=\(.input) asrolearn=\(.asrolearn)"')" +eval $(aws sts assume-role --role-arn $asrolearn --role-session-name awscli | jq -cr '"export AWS_ACCESS_KEY_ID=" + .Credentials.AccessKeyId, "export AWS_SECRET_ACCESS_KEY=" + .Credentials.SecretAccessKey, "export AWS_SESSION_TOKEN=" + .Credentials.SessionToken, "export AWS_SESSION_EXPIRATION=" + .Credentials.Expiration') aws cloudwatch list-metrics --namespace CWAgent --metric-name disk_inodes_free \ --dimensions Name=InstanceId,Value=$id Name=path,Value=/ | \ diff --git a/modules/ManagementGovernance/Monitoring.EC2/get-os-platform.sh b/modules/ManagementGovernance/Monitoring.EC2/get-os-platform.sh index 79aeb3d..f9de740 100755 --- a/modules/ManagementGovernance/Monitoring.EC2/get-os-platform.sh +++ b/modules/ManagementGovernance/Monitoring.EC2/get-os-platform.sh @@ -1,5 +1,6 @@ #!/bin/bash -eval "$(jq -r '@sh "id=\(.input)"')" +eval "$(jq -r '@sh "export id=\(.input) asrolearn=\(.asrolearn)"')" +eval $(aws sts assume-role --role-arn $asrolearn --role-session-name awscli | jq -cr '"export AWS_ACCESS_KEY_ID=" + .Credentials.AccessKeyId, "export AWS_SECRET_ACCESS_KEY=" + .Credentials.SecretAccessKey, "export AWS_SESSION_TOKEN=" + .Credentials.SessionToken, "export AWS_SESSION_EXPIRATION=" + .Credentials.Expiration') EC2OS=$(aws ec2 describe-instances --instance-ids $id | jq -r '.Reservations[].Instances[].PlatformDetails') diff --git a/modules/ManagementGovernance/Monitoring.EC2/main.tf b/modules/ManagementGovernance/Monitoring.EC2/main.tf index d880979..2709a82 100644 --- a/modules/ManagementGovernance/Monitoring.EC2/main.tf +++ b/modules/ManagementGovernance/Monitoring.EC2/main.tf @@ -1,5 +1,5 @@ resource "aws_cloudwatch_metric_alarm" "ec2-StatusCheckFailed_System" { - alarm_name = "${var.cw-alarm-prefix}:EC2:StatusCheckFailed_System:${var.ec2-instance-id}" + alarm_name = "${var.settings.StatusCheckFailed_System.ecccode}-EC2_${var.ec2-instance-id}-StatusCheckFailed_System" comparison_operator = var.settings.StatusCheckFailed_System.comparison_operator evaluation_periods = var.settings.StatusCheckFailed_System.evaluation_periods metric_name = "StatusCheckFailed_System" @@ -16,13 +16,10 @@ resource "aws_cloudwatch_metric_alarm" "ec2-StatusCheckFailed_System" { InstanceId = var.ec2-instance-id } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } resource "aws_cloudwatch_metric_alarm" "ec2-StatusCheckFailed_Instance" { - alarm_name = "${var.cw-alarm-prefix}:EC2:StatusCheckFailed_Instance:${var.ec2-instance-id}" + alarm_name = "${var.settings.StatusCheckFailed_Instance.ecccode}-EC2_${var.ec2-instance-id}-StatusCheckFailed_Instance" comparison_operator = var.settings.StatusCheckFailed_Instance.comparison_operator evaluation_periods = var.settings.StatusCheckFailed_Instance.evaluation_periods metric_name = "StatusCheckFailed_Instance" @@ -39,13 +36,10 @@ resource "aws_cloudwatch_metric_alarm" "ec2-StatusCheckFailed_Instance" { InstanceId = var.ec2-instance-id } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } resource "aws_cloudwatch_metric_alarm" "ec2-CPUUtilization" { - alarm_name = "${var.cw-alarm-prefix}:EC2:CPUUtilization:${var.ec2-instance-id}" + alarm_name = "${var.settings.CPUUtilization.ecccode}-EC2_${var.ec2-instance-id}-CPUUtilization" comparison_operator = var.settings.CPUUtilization.comparison_operator evaluation_periods = var.settings.CPUUtilization.evaluation_periods metric_name = "CPUUtilization" @@ -63,9 +57,6 @@ resource "aws_cloudwatch_metric_alarm" "ec2-CPUUtilization" { InstanceId = var.ec2-instance-id } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } # cwagent metrics @@ -74,46 +65,31 @@ data "aws_instance" "ec2-instance" { } # get instance OS +/* data "external" "ec2-os" { program = ["bash", "${path.module}/get-os-platform.sh"] query = { input = var.ec2-instance-id - } -} - -# Linux specific checks -# default cw agent uses mem_used_percent metric -/* -resource "aws_cloudwatch_metric_alarm" "ec2-mem_free" { - count = data.external.ec2-os.result.os == "Linux" ? 1 : 0 - alarm_name = "${var.cw-alarm-prefix}:EC2:mem_free:${var.ec2-instance-id}" - comparison_operator = "LessThanThreshold" - evaluation_periods = "2" - metric_name = "mem_free" - period = "900" - statistic = "Average" - threshold = var.threshold-mem_free - alarm_description = "EC2:mem_free" - namespace = "CWAgent" - insufficient_data_actions = [] - actions_enabled = var.actions-enabled - alarm_actions = [var.sns-targets.alarm-actions-standard] - ok_actions = [var.sns-targets.alarm-actions-standard] - dimensions = { - InstanceId = var.ec2-instance-id - ImageId = data.aws_instance.ec2-instance.ami - InstanceType = data.aws_instance.ec2-instance.instance_type - } - tags = var.default-tags - lifecycle { - ignore_changes = [tags] + asrolearn = var.asrolearn } } */ +module "ec2_os" { + source = "../../util/terraform-aws-cli" + assume_role_arn = var.asrolearn + role_session_name = "terraform-ec2-detect-os" + aws_cli_commands = ["ec2", "describe-instances", "--instance-ids", var.ec2-instance-id] + aws_cli_query = "Reservations[].Instances[].PlatformDetails" +} + +# Linux specific checks +# default cw agent uses mem_used_percent metric + resource "aws_cloudwatch_metric_alarm" "ec2-mem_used_percent" { - count = data.external.ec2-os.result.os == "Linux" ? 1 : 0 - alarm_name = "${var.cw-alarm-prefix}:EC2:mem_used_percent:${var.ec2-instance-id}" + # count = data.external.ec2-os.result.os == "Linux" ? 1 : 0 + count = flatten(module.ec2_os.result)[0] == "Windows" ? 0 : 1 + alarm_name = "${var.settings.mem_used_percent.ecccode}-EC2_${var.ec2-instance-id}-mem_used_percent" comparison_operator = var.settings.mem_used_percent.comparison_operator evaluation_periods = var.settings.mem_used_percent.evaluation_periods metric_name = "mem_used_percent" @@ -132,64 +108,20 @@ resource "aws_cloudwatch_metric_alarm" "ec2-mem_used_percent" { InstanceType = data.aws_instance.ec2-instance.instance_type } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } -# default cw agent uses swap_used_percent metric -/* -resource "aws_cloudwatch_metric_alarm" "ec2-swap_free" { - count = data.external.ec2-os.result.os == "Linux" ? 1 : 0 - # zero is fine as most ec2 instances are deployed without any swap - alarm_name = "${var.cw-alarm-prefix}:EC2:swap_free:${var.ec2-instance-id}" - comparison_operator = "LessThanThreshold" - evaluation_periods = "2" - threshold = var.threshold-swap_free - alarm_description = "EC2:swap_free" - insufficient_data_actions = [] - actions_enabled = var.actions-enabled - alarm_actions = [var.sns-targets.alarm-actions-standard] - ok_actions = [var.sns-targets.alarm-actions-standard] - metric_query { - id = "m1" - metric { - metric_name = "swap_free" - namespace = "CWAgent" - period = 900 - stat = "Average" - dimensions = { - InstanceId = var.ec2-instance-id - ImageId = data.aws_instance.ec2-instance.ami - InstanceType = data.aws_instance.ec2-instance.instance_type - } - } - } - metric_query { - id = "e1" - expression = "IF(m1==0, ${var.threshold-swap_free}, m1)" - label = "swap_free_if_not_zero" - return_data = "true" - } - - tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } -} -*/ - data "external" "cw-dimensions" { program = ["bash", "${path.module}/get-cwagent-dimensions.sh"] query = { - input = var.ec2-instance-id + input = var.ec2-instance-id + asrolearn = var.asrolearn } } - resource "aws_cloudwatch_metric_alarm" "ec2-swap_used_percent" { - count = data.external.ec2-os.result.os == "Linux" ? 1 : 0 - alarm_name = "${var.cw-alarm-prefix}:EC2:swap_used_percent:${var.ec2-instance-id}" + # count = data.external.ec2-os.result.os == "Linux" ? 1 : 0 + count = flatten(module.ec2_os.result)[0] == "Windows" ? 0 : 1 + alarm_name = "${var.settings.swap_used_percent.ecccode}-EC2_${var.ec2-instance-id}-swap_used_percent" comparison_operator = var.settings.swap_used_percent.comparison_operator evaluation_periods = var.settings.swap_used_percent.evaluation_periods metric_name = "swap_used_percent" @@ -208,40 +140,12 @@ resource "aws_cloudwatch_metric_alarm" "ec2-swap_used_percent" { InstanceType = data.aws_instance.ec2-instance.instance_type } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } -# default cw agent uses disk_used_percent metric -/* -resource "aws_cloudwatch_metric_alarm" "ec2-disk_free" { - count = data.external.ec2-os.result.os == "Linux" && length(data.external.cw-dimensions.result) > 0 ? 1 : 0 - alarm_name = "${var.cw-alarm-prefix}:EC2:disk_free:${var.ec2-instance-id}" - comparison_operator = "LessThanThreshold" - evaluation_periods = "2" - metric_name = "disk_free" - period = "900" - statistic = "Average" - threshold = var.threshold-disk_free - alarm_description = "EC2:disk_free" - namespace = "CWAgent" - insufficient_data_actions = [] - actions_enabled = var.actions-enabled - alarm_actions = [var.sns-targets.alarm-actions-urgent] - ok_actions = [var.sns-targets.alarm-actions-urgent] - dimensions = data.external.cw-dimensions.result - - tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } -} -*/ - resource "aws_cloudwatch_metric_alarm" "ec2-disk_used_percent" { - count = data.external.ec2-os.result.os == "Linux" && length(data.external.cw-dimensions.result) > 0 ? 1 : 0 - alarm_name = "${var.cw-alarm-prefix}:EC2:disk_used_percent:${var.ec2-instance-id}" + # count = data.external.ec2-os.result.os == "Linux" && data.external.cw-dimensions.result != null ? 1 : 0 + count = flatten(module.ec2_os.result)[0] == "Windows" && data.external.cw-dimensions.result != null ? 0 : 1 + alarm_name = "${var.settings.disk_used_percent.ecccode}-EC2_${var.ec2-instance-id}-disk_used_percent" comparison_operator = var.settings.disk_used_percent.comparison_operator evaluation_periods = var.settings.disk_used_percent.evaluation_periods metric_name = "disk_used_percent" @@ -257,15 +161,13 @@ resource "aws_cloudwatch_metric_alarm" "ec2-disk_used_percent" { dimensions = data.external.cw-dimensions.result tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } resource "aws_cloudwatch_metric_alarm" "ec2-disk_inodes_free" { - count = data.external.ec2-os.result.os == "Linux" && length(data.external.cw-dimensions.result) > 0 ? 1 : 0 - alarm_name = "${var.cw-alarm-prefix}:EC2:disk_inodes_free:${var.ec2-instance-id}" + # count = data.external.ec2-os.result.os == "Linux" && data.external.cw-dimensions.result != null ? 1 : 0 + count = flatten(module.ec2_os.result)[0] == "Windows" && data.external.cw-dimensions.result != null ? 0 : 1 + alarm_name = "${var.settings.disk_inodes_free.ecccode}-EC2_${var.ec2-instance-id}-disk_inodes_free" comparison_operator = var.settings.disk_inodes_free.comparison_operator evaluation_periods = var.settings.disk_inodes_free.evaluation_periods metric_name = "disk_inodes_free" @@ -290,15 +192,13 @@ resource "aws_cloudwatch_metric_alarm" "ec2-disk_inodes_free" { } */ tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } # process metric not published by default cw agent config resource "aws_cloudwatch_metric_alarm" "ec2-processes_total" { - count = data.external.ec2-os.result.os == "Linux" ? 1 : 0 - alarm_name = "${var.cw-alarm-prefix}:EC2:processes_total:${var.ec2-instance-id}" +# count = data.external.ec2-os.result.os == "Linux" ? 1 : 0 + count = flatten(module.ec2_os.result)[0] == "Windows" ? 0 : 1 + alarm_name = "${var.settings.processes_total.ecccode}-EC2_${var.ec2-instance-id}-processes_total" comparison_operator = var.settings.processes_total.comparison_operator evaluation_periods = var.settings.processes_total.evaluation_periods metric_name = "processes_total" @@ -317,16 +217,14 @@ resource "aws_cloudwatch_metric_alarm" "ec2-processes_total" { InstanceType = data.aws_instance.ec2-instance.instance_type } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } # Windows specific checks resource "aws_cloudwatch_metric_alarm" "ec2-MemoryCommittedPct" { - count = data.external.ec2-os.result.os == "Windows" ? 1 : 0 - alarm_name = "${var.cw-alarm-prefix}:EC2:MemoryCommittedPct:${var.ec2-instance-id}" + # count = data.external.ec2-os.result.os == "Windows" ? 1 : 0 + count = flatten(module.ec2_os.result)[0] == "Windows" ? 1 : 0 + alarm_name = "${var.settings.MemoryCommittedPct.ecccode}-EC2_${var.ec2-instance-id}-MemoryCommittedPct" comparison_operator = var.settings.MemoryCommittedPct.comparison_operator evaluation_periods = var.settings.MemoryCommittedPct.evaluation_periods metric_name = "Memory % Committed Bytes In Use" @@ -346,14 +244,12 @@ resource "aws_cloudwatch_metric_alarm" "ec2-MemoryCommittedPct" { InstanceType = data.aws_instance.ec2-instance.instance_type } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } resource "aws_cloudwatch_metric_alarm" "ec2-LogicalDiskFreePct" { - count = data.external.ec2-os.result.os == "Windows" ? 1 : 0 - alarm_name = "${var.cw-alarm-prefix}:EC2:LogicalDiskFreePct:${var.ec2-instance-id}" + # count = data.external.ec2-os.result.os == "Windows" ? 1 : 0 + count = flatten(module.ec2_os.result)[0] == "Windows" ? 1 : 0 + alarm_name = "${var.settings.LogicalDiskFreePct.ecccode}-EC2_${var.ec2-instance-id}-LogicalDiskFreePct" comparison_operator = var.settings.LogicalDiskFreePct.comparison_operator evaluation_periods = var.settings.LogicalDiskFreePct.evaluation_periods metric_name = "LogicalDisk % Free Space" @@ -374,7 +270,4 @@ resource "aws_cloudwatch_metric_alarm" "ec2-LogicalDiskFreePct" { InstanceType = data.aws_instance.ec2-instance.instance_type } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } \ No newline at end of file diff --git a/modules/ManagementGovernance/Monitoring.EC2/provider.tf b/modules/ManagementGovernance/Monitoring.EC2/provider.tf index 7b64cf5..933a1eb 100644 --- a/modules/ManagementGovernance/Monitoring.EC2/provider.tf +++ b/modules/ManagementGovernance/Monitoring.EC2/provider.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.36.1" + version = ">= 4.36.1" } } } diff --git a/modules/ManagementGovernance/Monitoring.EC2/variables.tf b/modules/ManagementGovernance/Monitoring.EC2/variables.tf index 39caea2..d2cce12 100644 --- a/modules/ManagementGovernance/Monitoring.EC2/variables.tf +++ b/modules/ManagementGovernance/Monitoring.EC2/variables.tf @@ -2,5 +2,5 @@ variable "cw-alarm-prefix" {} variable "actions-enabled" {} variable "ec2-instance-id" {} variable "settings" {} - +variable "asrolearn" {} variable "default-tags" {} \ No newline at end of file diff --git a/modules/ManagementGovernance/Monitoring.EKS/main.tf b/modules/ManagementGovernance/Monitoring.EKS/main.tf index 8e1487f..beb5427 100644 --- a/modules/ManagementGovernance/Monitoring.EKS/main.tf +++ b/modules/ManagementGovernance/Monitoring.EKS/main.tf @@ -2,7 +2,7 @@ resource "aws_cloudwatch_metric_alarm" "eks-pod_cpu_utilization" { for_each = toset(var.pod-names) - alarm_name = "${var.cw-alarm-prefix}:EKS:${var.cluster-name}:${each.value}:${var.settings.alarm1.metric}" + alarm_name = "${each.value["ecccode"]}:${var.cw-alarm-prefix}:EKS:${var.cluster-name}:${each.value}:${var.settings.alarm1.metric}" comparison_operator = var.settings.alarm1.comparison_operator evaluation_periods = var.settings.alarm1.evaluation_periods metric_name = var.settings.alarm1.metric @@ -21,15 +21,12 @@ resource "aws_cloudwatch_metric_alarm" "eks-pod_cpu_utilization" { "Namespace" = var.eks-namespace } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } resource "aws_cloudwatch_metric_alarm" "eks-pod_memory_utilization" { for_each = toset(var.pod-names) - alarm_name = "${var.cw-alarm-prefix}:EKS:${var.cluster-name}:${each.value}:${var.settings.alarm2.metric}" + alarm_name = "${each.value["ecccode"]}:${var.cw-alarm-prefix}:EKS:${var.cluster-name}:${each.value}:${var.settings.alarm2.metric}" comparison_operator = "GreaterThanThreshold" evaluation_periods = "3" metric_name = var.settings.alarm2.metric @@ -48,15 +45,12 @@ resource "aws_cloudwatch_metric_alarm" "eks-pod_memory_utilization" { "Namespace" = var.eks-namespace } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } resource "aws_cloudwatch_metric_alarm" "eks-pod_number_of_container_restarts" { for_each = toset(var.pod-names) - alarm_name = "${var.cw-alarm-prefix}:EKS:${var.cluster-name}:${each.value}:${var.settings.alarm3.metric}" + alarm_name = "${each.value["ecccode"]}:${var.cw-alarm-prefix}:EKS:${var.cluster-name}:${each.value}:${var.settings.alarm3.metric}" comparison_operator = "GreaterThanThreshold" evaluation_periods = "3" metric_name = var.settings.alarm3.metric @@ -75,7 +69,4 @@ resource "aws_cloudwatch_metric_alarm" "eks-pod_number_of_container_restarts" { "Namespace" = var.eks-namespace } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } diff --git a/modules/ManagementGovernance/Monitoring.EKS/provider.tf b/modules/ManagementGovernance/Monitoring.EKS/provider.tf index 7b64cf5..933a1eb 100644 --- a/modules/ManagementGovernance/Monitoring.EKS/provider.tf +++ b/modules/ManagementGovernance/Monitoring.EKS/provider.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.36.1" + version = ">= 4.36.1" } } } diff --git a/modules/ManagementGovernance/Monitoring.EMR/main.tf b/modules/ManagementGovernance/Monitoring.EMR/main.tf index 2beb0b0..478384d 100644 --- a/modules/ManagementGovernance/Monitoring.EMR/main.tf +++ b/modules/ManagementGovernance/Monitoring.EMR/main.tf @@ -1,6 +1,6 @@ resource "aws_cloudwatch_metric_alarm" "emr-alarms" { for_each = var.settings - alarm_name = "${var.cw-alarm-prefix}:EMR:${each.value["metric"]}:${var.job-flow-id}" + alarm_name = "${each.value["ecccode"]}-EMR_${var.job-flow-id}-${each.value["metric"]}" comparison_operator = each.value["comparison_operator"] evaluation_periods = each.value["evaluation_periods"] metric_name = each.value["metric"] @@ -17,32 +17,4 @@ resource "aws_cloudwatch_metric_alarm" "emr-alarms" { JobFlowId = var.job-flow-id } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } -} - -/* -resource "aws_cloudwatch_metric_alarm" "emr-CapacityRemainingGB" { - alarm_name = "${var.cw-alarm-prefix}:EMR:CapacityRemainingGB:${var.job-flow-id}" - comparison_operator = "LessThanThreshold" - evaluation_periods = "1" - metric_name = "CapacityRemainingGB" - period = "3600" - statistic = "Average" - threshold = var.settings.CapacityRemainingGB.threshold - alarm_description = "EMR:CapacityRemainingGB" - namespace = "AWS/ElasticMapReduce" - insufficient_data_actions = [] - actions_enabled = var.actions-enabled - alarm_actions = [var.settings.CapacityRemainingGB.action] - ok_actions = [var.settings.CapacityRemainingGB.action] - dimensions = { - JobFlowId = var.job-flow-id - } - tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } -} -*/ \ No newline at end of file +} \ No newline at end of file diff --git a/modules/ManagementGovernance/Monitoring.EMR/provider.tf b/modules/ManagementGovernance/Monitoring.EMR/provider.tf index 7b64cf5..933a1eb 100644 --- a/modules/ManagementGovernance/Monitoring.EMR/provider.tf +++ b/modules/ManagementGovernance/Monitoring.EMR/provider.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.36.1" + version = ">= 4.36.1" } } } diff --git a/modules/ManagementGovernance/Monitoring.EventBridge/main.tf b/modules/ManagementGovernance/Monitoring.EventBridge/main.tf index fbfe20c..71ec961 100644 --- a/modules/ManagementGovernance/Monitoring.EventBridge/main.tf +++ b/modules/ManagementGovernance/Monitoring.EventBridge/main.tf @@ -14,7 +14,7 @@ resource "aws_cloudwatch_event_rule" "EventRule" { PATTERN tags = var.default-tags lifecycle { - ignore_changes = [tags] + ignore_changes = [tags["LastModified"]] } } diff --git a/modules/ManagementGovernance/Monitoring.EventBridge/provider.tf b/modules/ManagementGovernance/Monitoring.EventBridge/provider.tf index 7b64cf5..933a1eb 100644 --- a/modules/ManagementGovernance/Monitoring.EventBridge/provider.tf +++ b/modules/ManagementGovernance/Monitoring.EventBridge/provider.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.36.1" + version = ">= 4.36.1" } } } diff --git a/modules/ManagementGovernance/Monitoring.Kafka/main.tf b/modules/ManagementGovernance/Monitoring.Kafka/main.tf index d455816..7bee57d 100644 --- a/modules/ManagementGovernance/Monitoring.Kafka/main.tf +++ b/modules/ManagementGovernance/Monitoring.Kafka/main.tf @@ -1,5 +1,5 @@ resource "aws_cloudwatch_metric_alarm" "Kafka-ZooKeeperRequestLatencyMsMean" { - alarm_name = "${var.cw-alarm-prefix}:Kafka:ZooKeeperRequestLatencyMsMean:${var.cluster-name}" + alarm_name = "${var.settings.ZooKeeperRequestLatencyMsMean.ecccode}-Kafka_${var.cluster-name}-ZooKeeperRequestLatencyMsMean" comparison_operator = var.settings.ZooKeeperRequestLatencyMsMean.comparison_operator evaluation_periods = var.settings.ZooKeeperRequestLatencyMsMean.evaluation_periods metric_name = "ZooKeeperRequestLatencyMsMean" @@ -16,9 +16,6 @@ resource "aws_cloudwatch_metric_alarm" "Kafka-ZooKeeperRequestLatencyMsMean" { "Cluster Name" = var.cluster-name } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } data "aws_msk_cluster" "msk-cluster" { @@ -29,22 +26,9 @@ data "aws_msk_broker_nodes" "msk-broker" { cluster_arn = data.aws_msk_cluster.msk-cluster.arn } -/* -output debug { - value = data.aws_msk_broker_nodes.msk-broker.node_info_list -} -*/ -/* -module "msk-brokers" { - source = "../../util/resource-list" - resource-type = "kafka-brokers" - query-input = data.aws_msk_cluster.msk-cluster.arn -} -*/ - resource "aws_cloudwatch_metric_alarm" "Kafka-CpuUserSystem" { for_each = toset([for i in data.aws_msk_broker_nodes.msk-broker.node_info_list[*].broker_id : tostring(i)]) - alarm_name = "${var.cw-alarm-prefix}:Kafka:CpuUsage:${var.cluster-name}-${each.value}" + alarm_name = "${var.settings.CpuUserSystem.ecccode}-Kafka_${var.cluster-name}-${each.value}-CpuUsage" comparison_operator = var.settings.CpuUserSystem.comparison_operator evaluation_periods = var.settings.CpuUserSystem.evaluation_periods threshold = var.settings.CpuUserSystem.threshold @@ -89,14 +73,11 @@ resource "aws_cloudwatch_metric_alarm" "Kafka-CpuUserSystem" { } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } resource "aws_cloudwatch_metric_alarm" "Kafka-KafkaDataLogsDiskUsed" { for_each = toset([for i in data.aws_msk_broker_nodes.msk-broker.node_info_list[*].broker_id : tostring(i)]) - alarm_name = "${var.cw-alarm-prefix}:Kafka:KafkaDataLogsDiskUsed:${var.cluster-name}-${each.value}" + alarm_name = "${var.settings.KafkaDataLogsDiskUsed.ecccode}-Kafka_${var.cluster-name}-${each.value}-KafkaDataLogsDiskUsed" comparison_operator = var.settings.KafkaDataLogsDiskUsed.comparison_operator evaluation_periods = var.settings.KafkaDataLogsDiskUsed.evaluation_periods metric_name = "KafkaDataLogsDiskUsed" @@ -114,14 +95,11 @@ resource "aws_cloudwatch_metric_alarm" "Kafka-KafkaDataLogsDiskUsed" { "Broker ID" = each.value } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } resource "aws_cloudwatch_metric_alarm" "Kafka-HeapMemoryAfterGC" { for_each = toset([for i in data.aws_msk_broker_nodes.msk-broker.node_info_list[*].broker_id : tostring(i)]) - alarm_name = "${var.cw-alarm-prefix}:Kafka:HeapMemoryAfterGC:${var.cluster-name}-${each.value}" + alarm_name = "${var.settings.HeapMemoryAfterGC.ecccode}-Kafka_${var.cluster-name}-${each.value}-HeapMemoryAfterGC" comparison_operator = var.settings.HeapMemoryAfterGC.comparison_operator evaluation_periods = var.settings.HeapMemoryAfterGC.evaluation_periods metric_name = "HeapMemoryAfterGC" @@ -139,8 +117,5 @@ resource "aws_cloudwatch_metric_alarm" "Kafka-HeapMemoryAfterGC" { "Broker ID" = each.value } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } diff --git a/modules/ManagementGovernance/Monitoring.Kafka/provider.tf b/modules/ManagementGovernance/Monitoring.Kafka/provider.tf index 7b64cf5..933a1eb 100644 --- a/modules/ManagementGovernance/Monitoring.Kafka/provider.tf +++ b/modules/ManagementGovernance/Monitoring.Kafka/provider.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.36.1" + version = ">= 4.36.1" } } } diff --git a/modules/ManagementGovernance/Monitoring.NGW/main.tf b/modules/ManagementGovernance/Monitoring.NGW/main.tf index 636303d..29028ed 100644 --- a/modules/ManagementGovernance/Monitoring.NGW/main.tf +++ b/modules/ManagementGovernance/Monitoring.NGW/main.tf @@ -1,6 +1,6 @@ resource "aws_cloudwatch_metric_alarm" "ngw-alarms" { for_each = var.settings - alarm_name = "${var.cw-alarm-prefix}:NGW:${each.value["metric"]}:${var.res-id}" + alarm_name = "${each.value["ecccode"]}-NGW_${var.res-id}-${each.value["metric"]}" comparison_operator = each.value["comparison_operator"] evaluation_periods = each.value["evaluation_periods"] metric_name = each.value["metric"] @@ -17,79 +17,4 @@ resource "aws_cloudwatch_metric_alarm" "ngw-alarms" { NatGatewayId = var.res-id } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } - - -/* -resource "aws_cloudwatch_metric_alarm" "ngw-ErrorPortAllocation" { - alarm_name = "${var.cw-alarm-prefix}:NGW:${var.settings.alarm1.metric}:${var.res-id}" - comparison_operator = "GreaterThanThreshold" - evaluation_periods = "2" - metric_name = var.settings.alarm1.metric - period = "300" - statistic = "Average" - threshold = var.settings.alarm1.threshold - alarm_description = "NGW:${var.settings.alarm1.metric}" - namespace = "AWS/NATGateway" - insufficient_data_actions = [] - actions_enabled = var.actions-enabled - alarm_actions = [var.settings.alarm1.action] - ok_actions = [var.settings.alarm1.action] - dimensions = { - NatGatewayId = var.res-id - } - tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } -} - -resource "aws_cloudwatch_metric_alarm" "ngw-ConnectionEstablishedCount" { - alarm_name = "${var.cw-alarm-prefix}:NGW:${var.settings.alarm2.metric}:${var.res-id}" - comparison_operator = "GreaterThanThreshold" - evaluation_periods = "2" - metric_name = var.settings.alarm2.metric - period = "300" - statistic = "Average" - threshold = var.settings.alarm2.threshold - alarm_description = "NGW:${var.settings.alarm2.metric}" - namespace = "AWS/NATGateway" - insufficient_data_actions = [] - actions_enabled = var.actions-enabled - alarm_actions = [var.settings.alarm2.action] - ok_actions = [var.settings.alarm2.action] - dimensions = { - NatGatewayId = var.res-id - } - tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } -} - -resource "aws_cloudwatch_metric_alarm" "ngw-PacketsDropCount" { - alarm_name = "${var.cw-alarm-prefix}:NGW:${var.settings.alarm3.metric}:${var.res-id}" - comparison_operator = "GreaterThanThreshold" - evaluation_periods = "2" - metric_name = var.settings.alarm3.metric - period = "300" - statistic = "Average" - threshold = var.settings.alarm3.threshold - alarm_description = "NGW:${var.settings.alarm3.metric}" - namespace = "AWS/NATGateway" - insufficient_data_actions = [] - actions_enabled = var.actions-enabled - alarm_actions = [var.settings.alarm3.action] - ok_actions = [var.settings.alarm3.action] - dimensions = { - NatGatewayId = var.res-id - } - tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } -} -*/ \ No newline at end of file diff --git a/modules/ManagementGovernance/Monitoring.NGW/provider.tf b/modules/ManagementGovernance/Monitoring.NGW/provider.tf index 7b64cf5..933a1eb 100644 --- a/modules/ManagementGovernance/Monitoring.NGW/provider.tf +++ b/modules/ManagementGovernance/Monitoring.NGW/provider.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.36.1" + version = ">= 4.36.1" } } } diff --git a/modules/ManagementGovernance/Monitoring.NLB/main.tf b/modules/ManagementGovernance/Monitoring.NLB/main.tf index 90c1de6..46623d2 100644 --- a/modules/ManagementGovernance/Monitoring.NLB/main.tf +++ b/modules/ManagementGovernance/Monitoring.NLB/main.tf @@ -6,17 +6,53 @@ data "external" "nlb-targetgroups" { } } */ +locals { + nlb-name = "net/${split("/", var.load-balancer)[2]}/${split("/", var.load-balancer)[3]}" +} +resource "aws_cloudwatch_metric_alarm" "nlb-TCP_Target_Reset_Count" { + alarm_name = "${var.settings.TCP_Target_Reset_Count.ecccode}-NLB_${local.nlb-name}-TCP_Target_Reset_Count" + comparison_operator = var.settings.TCP_Target_Reset_Count.comparison_operator + evaluation_periods = var.settings.TCP_Target_Reset_Count.evaluation_periods + metric_name = "TCP_Target_Reset_Count" + period = var.settings.TCP_Target_Reset_Count.period + statistic = var.settings.TCP_Target_Reset_Count.statistic + threshold = var.settings.TCP_Target_Reset_Count.threshold + alarm_description = "NLB:TCP_Target_Reset_Count" + namespace = "AWS/NetworkELB" + insufficient_data_actions = [] + actions_enabled = var.actions-enabled + alarm_actions = [var.settings.TCP_Target_Reset_Count.action] + ok_actions = [var.settings.TCP_Target_Reset_Count.action] + dimensions = { + LoadBalancer = local.nlb-name + } + tags = var.default-tags +} +/* module "nlb-targetgroups" { source = "../../util/resource-list" resource-type = "nlb-targetgroups" query-input = var.load-balancer + asrolearn = var.asrolearn +} +*/ + +// causes Rate exceeded error, maybe because of adaptive AWS_RETRY_MODE? + +module "nlb_tgs" { + assume_role_arn = var.asrolearn + role_session_name = "terraform-resource-list" + source = "../../util/terraform-aws-cli" + aws_cli_commands = ["elbv2", "describe-target-groups", "--load-balancer-arn", var.load-balancer] + aws_cli_query = "TargetGroups[*].TargetGroupArn" } resource "aws_cloudwatch_metric_alarm" "nlb-HealthyHostCount" { - for_each = module.nlb-targetgroups.result-set - alarm_name = "${var.cw-alarm-prefix}:NLBTG:HealthyHostCount:${split(":", each.value)[5]}" + # for_each = module.nlb-targetgroups.result-set + for_each = toset(flatten(module.nlb_tgs.result)) + alarm_name = "${var.settings.HealthHostCountMin.ecccode}-NLBTG_${split(":", each.value)[5]}-HealthyHostCount" comparison_operator = var.settings.HealthHostCountMin.comparison_operator evaluation_periods = var.settings.HealthHostCountMin.evaluation_periods metric_name = "HealthyHostCount" @@ -34,7 +70,4 @@ resource "aws_cloudwatch_metric_alarm" "nlb-HealthyHostCount" { LoadBalancer = "net/${split("/", var.load-balancer)[2]}/${split("/", var.load-balancer)[3]}" } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } diff --git a/modules/ManagementGovernance/Monitoring.NLB/outputs.tf b/modules/ManagementGovernance/Monitoring.NLB/outputs.tf new file mode 100644 index 0000000..93b1303 --- /dev/null +++ b/modules/ManagementGovernance/Monitoring.NLB/outputs.tf @@ -0,0 +1,4 @@ +output nlb-tg-count { + # value = length(module.nlb-targetgroups.result-set) + value = length(flatten(module.nlb_tgs.result)) +} \ No newline at end of file diff --git a/modules/ManagementGovernance/Monitoring.NLB/provider.tf b/modules/ManagementGovernance/Monitoring.NLB/provider.tf index 7b64cf5..933a1eb 100644 --- a/modules/ManagementGovernance/Monitoring.NLB/provider.tf +++ b/modules/ManagementGovernance/Monitoring.NLB/provider.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.36.1" + version = ">= 4.36.1" } } } diff --git a/modules/ManagementGovernance/Monitoring.NLB/variables.tf b/modules/ManagementGovernance/Monitoring.NLB/variables.tf index 13dac0a..902f979 100644 --- a/modules/ManagementGovernance/Monitoring.NLB/variables.tf +++ b/modules/ManagementGovernance/Monitoring.NLB/variables.tf @@ -2,4 +2,5 @@ variable cw-alarm-prefix {} variable actions-enabled {} variable load-balancer {} variable settings {} -variable default-tags {} \ No newline at end of file +variable default-tags {} +variable asrolearn {} \ No newline at end of file diff --git a/modules/ManagementGovernance/Monitoring.OpenSearch/main.tf b/modules/ManagementGovernance/Monitoring.OpenSearch/main.tf index ed152f0..470f707 100644 --- a/modules/ManagementGovernance/Monitoring.OpenSearch/main.tf +++ b/modules/ManagementGovernance/Monitoring.OpenSearch/main.tf @@ -2,7 +2,7 @@ data "aws_caller_identity" "this" {} resource "aws_cloudwatch_metric_alarm" "ES-alarms" { for_each = var.settings - alarm_name = "${var.cw-alarm-prefix}:ES:${each.value["metric"]}:${var.domain-name}" + alarm_name = "${each.value["ecccode"]}-ES_${var.domain-name}-${each.value["metric"]}" comparison_operator = each.value["comparison_operator"] evaluation_periods = each.value["evaluation_periods"] metric_name = each.value["metric"] @@ -20,7 +20,4 @@ resource "aws_cloudwatch_metric_alarm" "ES-alarms" { ClientId = data.aws_caller_identity.this.id } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } diff --git a/modules/ManagementGovernance/Monitoring.OpenSearch/provider.tf b/modules/ManagementGovernance/Monitoring.OpenSearch/provider.tf index 7b64cf5..933a1eb 100644 --- a/modules/ManagementGovernance/Monitoring.OpenSearch/provider.tf +++ b/modules/ManagementGovernance/Monitoring.OpenSearch/provider.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.36.1" + version = ">= 4.36.1" } } } diff --git a/modules/ManagementGovernance/Monitoring.RDS/README.md b/modules/ManagementGovernance/Monitoring.RDS/README.md index 11e162e..60a3027 100644 --- a/modules/ManagementGovernance/Monitoring.RDS/README.md +++ b/modules/ManagementGovernance/Monitoring.RDS/README.md @@ -3,6 +3,7 @@ This module deploys the default cloudwatch metric monitoring ## Notes Terraform lifecycle ignores tags to speed up terraform subsequent update. Cloudwatch alarm tags cannot be read on aws console anyway. +AWS provider 4.47.0 or above is needed for datasource aws_db_instances (https://github.com/hashicorp/terraform-provider-aws/blob/main/CHANGELOG.md) ## Example ```terraform diff --git a/modules/ManagementGovernance/Monitoring.RDS/main.tf b/modules/ManagementGovernance/Monitoring.RDS/main.tf index 4acb125..99059ad 100644 --- a/modules/ManagementGovernance/Monitoring.RDS/main.tf +++ b/modules/ManagementGovernance/Monitoring.RDS/main.tf @@ -1,6 +1,6 @@ resource "aws_cloudwatch_metric_alarm" "rds-alarms" { for_each = var.settings - alarm_name = "${var.cw-alarm-prefix}:RDS:${each.value["metric"]}:${var.rds-instance-name}" + alarm_name = "${each.value["ecccode"]}-RDS_${var.rds-instance-name}-${each.value["metric"]}" comparison_operator = each.value["comparison_operator"] evaluation_periods = each.value["evaluation_periods"] metric_name = each.value["metric"] @@ -17,7 +17,4 @@ resource "aws_cloudwatch_metric_alarm" "rds-alarms" { DBInstanceIdentifier = var.rds-instance-name } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } diff --git a/modules/ManagementGovernance/Monitoring.RDS/provider.tf b/modules/ManagementGovernance/Monitoring.RDS/provider.tf index 7b64cf5..9b051fa 100644 --- a/modules/ManagementGovernance/Monitoring.RDS/provider.tf +++ b/modules/ManagementGovernance/Monitoring.RDS/provider.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.36.1" + version = ">= 4.47.0" } } } diff --git a/modules/ManagementGovernance/Monitoring.Redis/main.tf b/modules/ManagementGovernance/Monitoring.Redis/main.tf index d48ef40..3504cd7 100644 --- a/modules/ManagementGovernance/Monitoring.Redis/main.tf +++ b/modules/ManagementGovernance/Monitoring.Redis/main.tf @@ -1,6 +1,6 @@ resource "aws_cloudwatch_metric_alarm" "redis-alarms" { for_each = var.settings - alarm_name = "${var.cw-alarm-prefix}:Redis:${each.value["metric"]}:${var.redis-cluster-id}" + alarm_name = "${each.value["ecccode"]}-Redis_${var.redis-cluster-id}-${each.value["metric"]}" comparison_operator = each.value["comparison_operator"] evaluation_periods = each.value["evaluation_periods"] metric_name = each.value["metric"] @@ -8,7 +8,7 @@ resource "aws_cloudwatch_metric_alarm" "redis-alarms" { statistic = each.value["statistic"] threshold = each.value["threshold"] alarm_description = "NGW:${each.value["metric"]}" - namespace = "AWS/NATGateway" + namespace = "AWS/ElastiCache" insufficient_data_actions = [] actions_enabled = var.actions-enabled alarm_actions = [each.value["action"]] @@ -17,7 +17,4 @@ resource "aws_cloudwatch_metric_alarm" "redis-alarms" { CacheClusterId = var.redis-cluster-id } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } \ No newline at end of file diff --git a/modules/ManagementGovernance/Monitoring.Redis/provider.tf b/modules/ManagementGovernance/Monitoring.Redis/provider.tf index 7b64cf5..933a1eb 100644 --- a/modules/ManagementGovernance/Monitoring.Redis/provider.tf +++ b/modules/ManagementGovernance/Monitoring.Redis/provider.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.36.1" + version = ">= 4.36.1" } } } diff --git a/modules/ManagementGovernance/Monitoring.TGW/main.tf b/modules/ManagementGovernance/Monitoring.TGW/main.tf index 5c21323..0337042 100644 --- a/modules/ManagementGovernance/Monitoring.TGW/main.tf +++ b/modules/ManagementGovernance/Monitoring.TGW/main.tf @@ -1,22 +1,20 @@ resource "aws_cloudwatch_metric_alarm" "tgw-PacketDropCountNoRoute" { - alarm_name = "${var.cw-alarm-prefix}:TGW:PacketDropCountNoRoute:${var.tgw-id}" - comparison_operator = var.settings.PacketDropCountNoRoute.comparison_operator - evaluation_periods = var.settings.PacketDropCountNoRoute.evaluation_periods - metric_name = "PacketDropCountNoRoute" - period = var.settings.PacketDropCountNoRoute.period - statistic = var.settings.PacketDropCountNoRoute.statistic - threshold = var.settings.PacketDropCountNoRoute.threshold - alarm_description = "TGW:PacketDropCountNoRoute" + for_each = var.settings + alarm_name = "${each.value["ecccode"]}-TGW_${var.tgw-id}-PacketDropCountNoRoute" + comparison_operator = each.value["comparison_operator"] + evaluation_periods = each.value["evaluation_periods"] + metric_name = each.value["metric"] + period = each.value["period"] + statistic = each.value["statistic"] + threshold = each.value["threshold"] + alarm_description = "TGW:${each.value["metric"]}" namespace = "AWS/TransitGateway" insufficient_data_actions = [] actions_enabled = var.actions-enabled - alarm_actions = [var.settings.PacketDropCountNoRoute.action] - ok_actions = [var.settings.PacketDropCountNoRoute.action] + alarm_actions = [each.value["action"]] + ok_actions = [each.value["action"]] dimensions = { TransitGateway = var.tgw-id } tags = var.default-tags - lifecycle { - ignore_changes = [tags] - } } \ No newline at end of file diff --git a/modules/ManagementGovernance/Monitoring.TGW/provider.tf b/modules/ManagementGovernance/Monitoring.TGW/provider.tf index 7b64cf5..9b051fa 100644 --- a/modules/ManagementGovernance/Monitoring.TGW/provider.tf +++ b/modules/ManagementGovernance/Monitoring.TGW/provider.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.36.1" + version = ">= 4.47.0" } } } diff --git a/modules/util/resource-list/README.md b/modules/util/resource-list/README.md new file mode 100644 index 0000000..14c4130 --- /dev/null +++ b/modules/util/resource-list/README.md @@ -0,0 +1,2 @@ +# resource-list module +Module for listing resources, where native terraform data source is not available. \ No newline at end of file diff --git a/modules/util/resource-list/list-alb-targetgroups.sh b/modules/util/resource-list/list-alb-targetgroups.sh new file mode 100755 index 0000000..60dc5c8 --- /dev/null +++ b/modules/util/resource-list/list-alb-targetgroups.sh @@ -0,0 +1,6 @@ +#!/bin/bash +eval "$(jq -r '@sh "export lb=\(.input) asrolearn=\(.asrolearn)"')" +eval "$(aws sts assume-role --role-arn $asrolearn --role-session-name awscli | jq -cr '"export AWS_ACCESS_KEY_ID=" + .Credentials.AccessKeyId, "export AWS_SECRET_ACCESS_KEY=" + .Credentials.SecretAccessKey, "export AWS_SESSION_TOKEN=" + .Credentials.SessionToken')" + +RESULTS=$(aws elbv2 describe-target-groups --load-balancer-arn $lb --query TargetGroups[*].TargetGroupArn --output text --no-cli-pager | sed 's/\t/\n/g' | sort | xargs) +jq -n --arg result "$RESULTS" '{"result":$result}' \ No newline at end of file diff --git a/modules/util/resource-list/list-emr.sh b/modules/util/resource-list/list-emr.sh index 6a6bef2..3bec841 100755 --- a/modules/util/resource-list/list-emr.sh +++ b/modules/util/resource-list/list-emr.sh @@ -1,3 +1,6 @@ #!/bin/bash +eval "$(jq -r '@sh "asrolearn=\(.asrolearn)"')" +eval "$(aws sts assume-role --role-arn $asrolearn --role-session-name awscli | jq -cr '"export AWS_ACCESS_KEY_ID=" + .Credentials.AccessKeyId, "export AWS_SECRET_ACCESS_KEY=" + .Credentials.SecretAccessKey, "export AWS_SESSION_TOKEN=" + .Credentials.SessionToken')" + RESULTS=$(aws emr list-clusters --active --query Clusters[*].ClusterArn --output text --no-cli-pager | sed 's/\t/\n/g' | sort | xargs) jq -n --arg result "$RESULTS" '{"result":$result}' diff --git a/modules/util/resource-list/list-kafka.sh b/modules/util/resource-list/list-kafka.sh index 5e92761..a6f30b5 100755 --- a/modules/util/resource-list/list-kafka.sh +++ b/modules/util/resource-list/list-kafka.sh @@ -1,5 +1,7 @@ #!/bin/bash -# exclude ASG instances +eval "$(jq -r '@sh "asrolearn=\(.asrolearn)"')" +eval "$(aws sts assume-role --role-arn $asrolearn --role-session-name awscli | jq -cr '"export AWS_ACCESS_KEY_ID=" + .Credentials.AccessKeyId, "export AWS_SECRET_ACCESS_KEY=" + .Credentials.SecretAccessKey, "export AWS_SESSION_TOKEN=" + .Credentials.SessionToken')" + RESULTS=$(aws kafka list-clusters --query ClusterInfoList[*].ClusterName --output text --no-cli-pager | sed 's/\t/\n/g' | sort | xargs) jq -n --arg result "$RESULTS" '{"result":$result}' diff --git a/modules/util/resource-list/list-nlb-targetgroups.sh b/modules/util/resource-list/list-nlb-targetgroups.sh index bfea521..6b6070b 100755 --- a/modules/util/resource-list/list-nlb-targetgroups.sh +++ b/modules/util/resource-list/list-nlb-targetgroups.sh @@ -1,5 +1,6 @@ #!/bin/bash -eval "$(jq -r '@sh "lb=\(.input)"')" +eval "$(jq -r '@sh "export lb=\(.input) asrolearn=\(.asrolearn)"')" +eval "$(aws sts assume-role --role-arn $asrolearn --role-session-name awscli | jq -cr '"export AWS_ACCESS_KEY_ID=" + .Credentials.AccessKeyId, "export AWS_SECRET_ACCESS_KEY=" + .Credentials.SecretAccessKey, "export AWS_SESSION_TOKEN=" + .Credentials.SessionToken')" RESULTS=$(aws elbv2 describe-target-groups --load-balancer-arn $lb --query TargetGroups[*].TargetGroupArn --output text --no-cli-pager | sed 's/\t/\n/g' | sort | xargs) jq -n --arg result "$RESULTS" '{"result":$result}' diff --git a/modules/util/resource-list/list-opensearch.sh b/modules/util/resource-list/list-opensearch.sh index 65882c1..8f4472b 100755 --- a/modules/util/resource-list/list-opensearch.sh +++ b/modules/util/resource-list/list-opensearch.sh @@ -1,5 +1,7 @@ #!/bin/bash -# exclude ASG instances +eval "$(jq -r '@sh "asrolearn=\(.asrolearn)"')" +eval "$(aws sts assume-role --role-arn $asrolearn --role-session-name awscli | jq -cr '"export AWS_ACCESS_KEY_ID=" + .Credentials.AccessKeyId, "export AWS_SECRET_ACCESS_KEY=" + .Credentials.SecretAccessKey, "export AWS_SESSION_TOKEN=" + .Credentials.SessionToken')" + RESULTS=$(aws opensearch list-domain-names --query DomainNames[*].DomainName --output text --no-cli-pager | sed 's/\t/\n/g' | sort | xargs) jq -n --arg result "$RESULTS" '{"result":$result}' diff --git a/modules/util/resource-list/list-redis.sh b/modules/util/resource-list/list-redis.sh index 981b00a..82390c4 100644 --- a/modules/util/resource-list/list-redis.sh +++ b/modules/util/resource-list/list-redis.sh @@ -1,3 +1,6 @@ #!/bin/bash +eval "$(jq -r '@sh "asrolearn=\(.asrolearn)"')" +eval "$(aws sts assume-role --role-arn $asrolearn --role-session-name awscli | jq -cr '"export AWS_ACCESS_KEY_ID=" + .Credentials.AccessKeyId, "export AWS_SECRET_ACCESS_KEY=" + .Credentials.SecretAccessKey, "export AWS_SESSION_TOKEN=" + .Credentials.SessionToken')" + RESULTS=$( aws elasticache describe-cache-clusters --query 'CacheClusters[*].CacheClusterId' --output text --no-cli-pager | sed 's/\t/\n/g' | sort | xargs) jq -n --arg result "$RESULTS" '{"result":$result}' diff --git a/modules/util/resource-list/list-tgw.sh b/modules/util/resource-list/list-tgw.sh index d32edb4..d37abaf 100755 --- a/modules/util/resource-list/list-tgw.sh +++ b/modules/util/resource-list/list-tgw.sh @@ -1,3 +1,7 @@ #!/bin/bash + +eval "$(jq -r '@sh "asrolearn=\(.asrolearn)"')" +eval "$(aws sts assume-role --role-arn $asrolearn --role-session-name awscli | jq -cr '"export AWS_ACCESS_KEY_ID=" + .Credentials.AccessKeyId, "export AWS_SECRET_ACCESS_KEY=" + .Credentials.SecretAccessKey, "export AWS_SESSION_TOKEN=" + .Credentials.SessionToken')" + RESULTS=$(aws ec2 describe-transit-gateways --query 'TransitGateways[].TransitGatewayId' --output text --no-cli-pager | sed 's/\t/\n/g' | sort | xargs) jq -n --arg result "$RESULTS" '{"result":$result}' \ No newline at end of file diff --git a/modules/util/resource-list/main.tf b/modules/util/resource-list/main.tf index e0a0ac7..0605f20 100644 --- a/modules/util/resource-list/main.tf +++ b/modules/util/resource-list/main.tf @@ -1,8 +1,8 @@ data "external" "instances" { - # program = ["bash", "../../modules/util/resource-list/list-${var.resource-type}.sh"] program = ["bash", "${path.module}/list-${var.resource-type}.sh"] query = { input = var.query-input + asrolearn = var.asrolearn } } diff --git a/modules/util/resource-list/variables.tf b/modules/util/resource-list/variables.tf index 6a21820..5003682 100644 --- a/modules/util/resource-list/variables.tf +++ b/modules/util/resource-list/variables.tf @@ -5,4 +5,13 @@ variable resource-type { variable query-input { type = string default = null +} + +variable asrolearn { + type = string + + validation { + condition = length(var.asrolearn) > 1 + error_message = "asrolearn is too short" + } } \ No newline at end of file diff --git a/modules/util/terraform-aws-cli/.gitignore b/modules/util/terraform-aws-cli/.gitignore new file mode 100644 index 0000000..680685a --- /dev/null +++ b/modules/util/terraform-aws-cli/.gitignore @@ -0,0 +1,5 @@ +/test-reports/ +.idea/ +/PersonalSettingsMakefile +.terraform/ +/temp/ diff --git a/modules/util/terraform-aws-cli/.pre-commit-config.yaml b/modules/util/terraform-aws-cli/.pre-commit-config.yaml new file mode 100644 index 0000000..c41a49e --- /dev/null +++ b/modules/util/terraform-aws-cli/.pre-commit-config.yaml @@ -0,0 +1,43 @@ +repos: + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.77.2 + hooks: + - id: terraform_tflint + - id: terraform_fmt + - id: terraform_validate + exclude: modules + - id: terraform_docs + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-added-large-files + - id: check-executables-have-shebangs + - id: check-json + - id: check-merge-conflict + - id: check-symlinks + - id: check-yaml + - id: detect-aws-credentials + args: + - --allow-missing-credentials + - id: detect-private-key + - id: end-of-file-fixer + - id: fix-byte-order-marker + - id: pretty-format-json + files: .*\.json$ + args: + - --autofix + - --indent=2 + - --no-sort-keys + - id: trailing-whitespace + + - repo: https://github.com/jumanjihouse/pre-commit-hook-yamlfmt + rev: 0.2.2 + hooks: + - id: yamlfmt + args: + - --implicit_start + - --preserve-quotes + - --mapping=2 + - --offset=2 + - --sequence=4 + - --width=300 diff --git a/modules/util/terraform-aws-cli/.terraform-version b/modules/util/terraform-aws-cli/.terraform-version new file mode 100644 index 0000000..e516bb9 --- /dev/null +++ b/modules/util/terraform-aws-cli/.terraform-version @@ -0,0 +1 @@ +1.4.5 diff --git a/modules/util/terraform-aws-cli/.tflint.hcl b/modules/util/terraform-aws-cli/.tflint.hcl new file mode 100644 index 0000000..49299ef --- /dev/null +++ b/modules/util/terraform-aws-cli/.tflint.hcl @@ -0,0 +1,32 @@ +config { + module = true + force = false +} + +// Only the AWS plugin is enabled. The Google and Azure plugins are not enabled as we have no current use for them. +plugin "aws" { + enabled = true + source = "github.com/terraform-linters/tflint-ruleset-aws" + version = "0.22.1" + deep_check = true +} + +rule "terraform_naming_convention" { + enabled = true +} + +rule "terraform_deprecated_interpolation" { + enabled = true +} + +rule "terraform_documented_outputs" { + enabled = true +} + +rule "terraform_documented_variables" { + enabled = true +} + +rule "terraform_module_pinned_source" { + enabled = true +} diff --git a/modules/util/terraform-aws-cli/.travis.yml b/modules/util/terraform-aws-cli/.travis.yml new file mode 100644 index 0000000..505891c --- /dev/null +++ b/modules/util/terraform-aws-cli/.travis.yml @@ -0,0 +1,12 @@ +install: + - sudo apt-get -y install jq + - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + - unzip awscliv2.zip + - sudo ./aws/install + - git clone https://github.com/tfutils/tfenv.git ~/.tfenv + - sudo ln -s ~/.tfenv/bin/* /usr/local/bin + - tfenv install + +script: + - terraform init + - tests/tests.sh diff --git a/modules/util/terraform-aws-cli/CHANGELOG.md b/modules/util/terraform-aws-cli/CHANGELOG.md new file mode 100644 index 0000000..0a2b75e --- /dev/null +++ b/modules/util/terraform-aws-cli/CHANGELOG.md @@ -0,0 +1,86 @@ +# Changelog + +# v5.0.4 - 2022/11/28 + +- Allow `var.role_session_name` to be optional. Thank you [Byron Kim](https://github.com/digitickets/terraform-aws-cli/issues/4) + +# v5.0.3 - 2022/05/31 + +- Fix for when the AWS call being made has no output (which is invalid JSON). Thank you [Yaron Yarimi and Pavel Kargin](https://github.com/digitickets/terraform-aws-cli/issues/3) + +# v5.0.2 - 2022/05/26 + +- Fix for when this module is used in an iteration. + +# v5.0.1 - 2022/05/24 + +- Explicitly specify output type as json for assume role call. Thank you [Niranjan Rajendran](https://github.com/digitickets/terraform-aws-cli/pull/2) + +# v5.0.0 - 2022/01/27 + +- Fixed incompatibilities with Terraform 1.1.0. + +# v4.1.0 - 2021/10/05 + +- Validate role_session_name so that the maximum length is 64 characters and that it must match a specific regex. + +# v4.0.0 - 2021/05/18 + +- Set minimum terraform version to 0.15.0. + +# No release required - 2021/03/30 + +- Updated tests to use an AWS request that does not require credentials, allowing the full terraform plan and apply + process to be run and tested with the module. + +# v3.1.1 - 2021/03/25 + +- Re-releasing as accidentally released v3.0.0 as v3.1.0. + +# v3.1.0 - 2021/03/25 + +- Add an optional `debug_log_filename` variable. If supplied, a log file will be produced in the supplied location. This + option enables the `--debug` option of the AWS CLI. Use this in safe environments as potentially sensitive content may + be logged. +- Added [adaptive retry mode](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-retries.html#cli-usage-retries-modes-adaptive) + to help alleviate throttling issues. + +# v3.0.0 - 2020/12/03 + +- Set minimum terraform version to 0.14.0. +- Introduced `.terraform.lock.hcl` for versioning of dependencies. + +# v2.0.1 - 2020/09/17 + +- Add `depends_on` to enforce the order in which the resources get instantiated / evaluated. + +# v2.0.0 - 2020/09/17 + +- Set minimum terraform version to 0.13.0 +- Added variable validation to optional `assume_role_arn` to match syntax described in + [IAM Identifiers](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html). + +# v1.3.0 - 2020/08/03 + +- Set minimum version of random provider to 2.3.0 + +# v1.2.2 - 2020/05/11 + +- Updated examples in [README.md](README.md). + +# v1.2.1 - 2020/05/11 + +- Updated [README.md](README.md) to reflect `digiticketsgroup/terraforming` image that includes all the required + resources for using this module. + +# v1.2.0 - 2020/05/11 + +- Drop down to using `sh` rather than `bash` so this module can operate with Hashicorp Terraform Docker image. + +# v1.1.0 - 2020/05/07 + +- Updated examples in README.md with registry path as displayed by registry. +- Updated `assume_role_arn` to reflect that it is optional. + +# v1.0.0 - 2020/05/07 +Initial release diff --git a/modules/util/terraform-aws-cli/README.md b/modules/util/terraform-aws-cli/README.md new file mode 100644 index 0000000..40b72a0 --- /dev/null +++ b/modules/util/terraform-aws-cli/README.md @@ -0,0 +1,131 @@ +[![Build Status](https://img.shields.io/travis/digitickets/terraform-aws-cli.svg?style=for-the-badge&logo=travis)](https://travis-ci.com/digitickets/terraform-aws-cli) +[![GitHub issues](https://img.shields.io/github/issues/digitickets/terraform-aws-cli.svg?style=for-the-badge&logo=github)](https://github.com/digitickets/terraform-aws-cli/issues) + +# terraform-aws-cli + +Run the AWS CLI, with the ability to run under an assumed role, to access resources and properties missing from the +Terraform AWS Provider. + +# Requirements + +This module requires a couple of additional resources to operate successfully. + +1. Amazon Web Service Command Line Interface (awscli) + : This is available in several forms [here](https://aws.amazon.com/cli/). + +2. JSON processor (jq) + : This is available [here](https://stedolan.github.io/jq/). + +# Examples + +## 1. Get the desired capacity of an autoscaling group. + +If you are using a blue/green style deployment, you would want to create the same number of EC2 instances as you are +replacing. + +```hcl-terraform +module "current_desired_capacity" { + source = "digitickets/cli/aws" + role_session_name = "GettingDesiredCapacityFor${var.environment}" + aws_cli_commands = ["autoscaling", "describe-auto-scaling-groups"] + aws_cli_query = "AutoScalingGroups[?Tags[?Key==`Name`]|[?Value==`digitickets-${var.environment}-asg-app`]]|[0].DesiredCapacity" +} +``` + +You can now set the desired capacity of an aws_autoscaling_group: + +```hcl-terraform + desired_capacity = module.current_desired_capacity.result +``` + +## 2. Assuming a role. + +Extending the first example above, assuming a role is as simple as adding an `assume_role_arn` to the module: + +```hcl-terraform +module "current_desired_capacity" { + source = "digitickets/cli/aws" + assume_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/OrganizationAccountAccessRole" + role_session_name = "GettingDesiredCapacityFor${var.environment}" + aws_cli_commands = ["autoscaling", "describe-auto-scaling-groups"] + aws_cli_query = "AutoScalingGroups[?Tags[?Key==`Name`]|[?Value==`digitickets-${var.environment}-asg-app`]]|[0].DesiredCapacity" +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15 | +| [external](#requirement\_external) | ~> 2.0 | +| [local](#requirement\_local) | ~> 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [external](#provider\_external) | 2.3.1 | +| [local](#provider\_local) | 2.4.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [external_external.awscli_program](https://registry.terraform.io/providers/hashicorp/external/latest/docs/data-sources/external) | data source | +| [local_file.awscli_results_file](https://registry.terraform.io/providers/hashicorp/local/latest/docs/data-sources/file) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [assume\_role\_arn](#input\_assume\_role\_arn) | The ARN of the role being assumed (optional) | `string` | `""` | no | +| [aws\_cli\_commands](#input\_aws\_cli\_commands) | The AWS CLI command and subcommands | `list(string)` | n/a | yes | +| [aws\_cli\_query](#input\_aws\_cli\_query) | The --query value | `string` | `""` | no | +| [debug\_log\_filename](#input\_debug\_log\_filename) | Generate a debug log if a `debug_log_filename` is supplied | `string` | `""` | no | +| [role\_session\_name](#input\_role\_session\_name) | The role session name | `string` | `""` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [result](#output\_result) | The output of the AWS CLI command | + + +# Docker + +To help with getting this running in a pipeline that uses Docker, the image [digiticketsgroup/terraforming](https://hub.docker.com/repository/docker/digiticketsgroup/terraforming) has Terraform, AWSCLI, and jq all ready to go. + +If you want to build or adapt your own image, then the Dockerfile below is how that image has been built. + +```Dockerfile +# Based upon https://github.com/aws/aws-cli/blob/2.0.10/docker/Dockerfile +FROM amazonlinux:2 as installer +ARG TERRAFORM_VERSION +RUN yum update -y \ + && yum install -y unzip \ + && curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscli-exe-linux-x86_64.zip \ + && unzip awscli-exe-linux-x86_64.zip \ + # The --bin-dir is specified so that we can copy the + # entire bin directory from the installer stage into + # into /usr/local/bin of the final stage without + # accidentally copying over any other executables that + # may be present in /usr/local/bin of the installer stage. + && ./aws/install --bin-dir /aws-cli-bin/ \ + && curl "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" -o terraform.zip \ + && unzip terraform.zip + +FROM amazonlinux:2 +COPY --from=installer /usr/local/aws-cli/ /usr/local/aws-cli/ +COPY --from=installer /aws-cli-bin/ /usr/local/bin/ +COPY --from=installer terraform /usr/bin/ +RUN yum update -y \ + && yum install -y less groff jq \ + && yum clean all + +ENTRYPOINT ["/bin/sh"] +``` diff --git a/modules/util/terraform-aws-cli/main.tf b/modules/util/terraform-aws-cli/main.tf new file mode 100644 index 0000000..8f58642 --- /dev/null +++ b/modules/util/terraform-aws-cli/main.tf @@ -0,0 +1,42 @@ +locals { + joined_aws_cli_command = join(" ", var.aws_cli_commands) + output_file = format( + "%s/temp/results-%s.json", + path.module, + md5( + join( + "-", + [ + var.assume_role_arn, + var.role_session_name, + local.joined_aws_cli_command, + var.aws_cli_query, + var.debug_log_filename + ] + ) + ) + ) +} + +data "external" "awscli_program" { + program = [format("%s/scripts/awsWithAssumeRole.sh", path.module)] + query = { + assume_role_arn = var.assume_role_arn + role_session_name = var.role_session_name + aws_cli_commands = local.joined_aws_cli_command + aws_cli_query = var.aws_cli_query + output_file = local.output_file + debug_log_filename = var.debug_log_filename + } +} + +data "local_file" "awscli_results_file" { + depends_on = [data.external.awscli_program] + filename = data.external.awscli_program.query.output_file +} + +output "result" { + depends_on = [data.local_file.awscli_results_file] + description = "The output of the AWS CLI command" + value = try(jsondecode(data.local_file.awscli_results_file.content), null) +} diff --git a/modules/util/terraform-aws-cli/scripts/awsWithAssumeRole.sh b/modules/util/terraform-aws-cli/scripts/awsWithAssumeRole.sh new file mode 100755 index 0000000..5fedb06 --- /dev/null +++ b/modules/util/terraform-aws-cli/scripts/awsWithAssumeRole.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env sh + +# Validate required commands +if ! [ -x "$(command -v aws)" ]; then + echo 'Error: aws is not installed.' >&2 + exit 1 +fi +if ! [ -x "$(command -v jq)" ]; then + echo 'Error: jq is not installed.' >&2 + exit 1 +fi + +# Get the query +TERRAFORM_QUERY=$(jq -Mc .) + +# Extract the query attributes +AWS_CLI_COMMANDS=$(echo "${TERRAFORM_QUERY}" | jq -r '.aws_cli_commands') +AWS_CLI_QUERY=$(echo "${TERRAFORM_QUERY}" | jq -r '.aws_cli_query') +OUTPUT_FILE=$(echo "${TERRAFORM_QUERY}" | jq -r '.output_file') +ASSUME_ROLE_ARN=$(echo "${TERRAFORM_QUERY}" | jq -r '.assume_role_arn') +ROLE_SESSION_NAME=$(echo "${TERRAFORM_QUERY}" | jq -r '.role_session_name') +DEBUG_LOG_FILENAME=$(echo "${TERRAFORM_QUERY}" | jq -r '.debug_log_filename') + +# Do we need to assume a role? +if [ -n "${ASSUME_ROLE_ARN}" ]; then + TEMP_ROLE=$(aws sts assume-role --output json --role-arn "${ASSUME_ROLE_ARN}" --role-session-name "${ROLE_SESSION_NAME:-AssumingRole}") + export AWS_ACCESS_KEY_ID=$(echo "${TEMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + export AWS_SECRET_ACCESS_KEY=$(echo "${TEMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + export AWS_SESSION_TOKEN=$(echo "${TEMP_ROLE}" | jq -r '.Credentials.SessionToken') +fi + +# Do we have a query? +if [ -n "${AWS_CLI_QUERY}" ]; then + AWS_CLI_QUERY_PARAM="--query '${AWS_CLI_QUERY}'" +fi + +# Do we want to be debug? +export AWS_DEBUG_OPTION="" +if [ -n "${DEBUG_LOG_FILENAME}" ]; then + AWS_DEBUG_OPTION="--debug 2>${DEBUG_LOG_FILENAME}" + mkdir -p "$(dirname ${DEBUG_LOG_FILENAME})" +fi + +# Make sure output file directory exists +mkdir -p "$(dirname ${OUTPUT_FILE})" + +# Make sure output file does not exist +rm -f "${OUTPUT_FILE}" + +# Disable any assigned pager +export AWS_PAGER="" + +# Configure adaptive retry mode +# export AWS_RETRY_MODE=adaptive +export AWS_RETRY_MODE=standard +export AWS_MAX_ATTEMPTS=3 + +# Run the AWS_CLI command, exiting with a non zero exit code if required. +if ! eval "aws ${AWS_CLI_COMMANDS} ${AWS_CLI_QUERY_PARAM:-} --output json ${AWS_DEBUG_OPTION}" >"${OUTPUT_FILE}" ; then + echo "Error: aws failed." + exit 1 +fi + +# All is good. +echo '{"output_file":"'"${OUTPUT_FILE}"'"}' diff --git a/modules/util/terraform-aws-cli/tests/bad_arn/terraform.tfvars b/modules/util/terraform-aws-cli/tests/bad_arn/terraform.tfvars new file mode 100644 index 0000000..0c1d712 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/bad_arn/terraform.tfvars @@ -0,0 +1,3 @@ +assume_role_arn = "bad_arn" +aws_cli_commands = ["version"] +role_session_name = "bad_arn" diff --git a/modules/util/terraform-aws-cli/tests/bad_arn/test.sh b/modules/util/terraform-aws-cli/tests/bad_arn/test.sh new file mode 100755 index 0000000..a71977f --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/bad_arn/test.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +function run_test() { +if [[ -f $PLAN_FILE ]]; then + echo "Incorrectly generated a plan - $PLAN_FILE"; + exit 1; +fi + +if [[ ! -z "$(cat $PLAN_LOG_FILE)" ]]; then + echo "Incorrectly generated content in the plan log file - $PLAN_LOG_FILE"; + exit 2; +fi + +if [[ ! "$(cat $PLAN_ERROR_FILE)" == *'The optional ARN must match the format documented in'* ]]; then + echo 'Failed to detect invalid ARN.'; + exit 3; +fi +} + +. tests/common.sh $0 diff --git a/modules/util/terraform-aws-cli/tests/common.sh b/modules/util/terraform-aws-cli/tests/common.sh new file mode 100644 index 0000000..c8f46b1 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/common.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +TEST_PATH=$(dirname $1) +TEST_NAME=$(basename $TEST_PATH) + +echo "Start : $TEST_PATH" + +TERRAFORM_TFVARS=$TEST_PATH/terraform.tfvars +EXPECTED_VARIABLES=$TEST_PATH/expected_variables.json + +RESOURCE_PATH=test-reports/$TEST_NAME +mkdir -p $RESOURCE_PATH + +INIT_LOG_FILE=$RESOURCE_PATH/init.log +INIT_ERROR_FILE=$RESOURCE_PATH/init.error.log +PLAN_FILE=$RESOURCE_PATH/terraform.plan +PLAN_LOG_FILE=$RESOURCE_PATH/plan.log +PLAN_ERROR_FILE=$RESOURCE_PATH/plan.error.log +STATE_FILE=$RESOURCE_PATH/terraform.tfstate +APPLY_LOG_FILE=$RESOURCE_PATH/apply.log +APPLY_ERROR_FILE=$RESOURCE_PATH/apply.error.log +DEBUG_LOG_FILE=$RESOURCE_PATH/debug.log + +terraform init > $INIT_LOG_FILE 2> $INIT_ERROR_FILE + +terraform plan -var-file=$TERRAFORM_TFVARS -out=$PLAN_FILE > $PLAN_LOG_FILE 2> $PLAN_ERROR_FILE + +run_test + +echo "Passed : $TEST_PATH" diff --git a/modules/util/terraform-aws-cli/tests/empty_result/expected_variables.json b/modules/util/terraform-aws-cli/tests/empty_result/expected_variables.json new file mode 100644 index 0000000..e9556b9 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/empty_result/expected_variables.json @@ -0,0 +1,24 @@ +{ + "assume_role_arn": { + "value": "" + }, + "aws_cli_commands": { + "value": [ + "guardduty", + "update-detector", + "--finding-publishing-frequency", + "ONE_HOUR", + "--detector-id", + "0123456789abcdef0123456789abcdef" + ] + }, + "aws_cli_query": { + "value": "" + }, + "debug_log_filename": { + "value": "" + }, + "role_session_name": { + "value": "empty_result" + } +} diff --git a/modules/util/terraform-aws-cli/tests/empty_result/notes.txt b/modules/util/terraform-aws-cli/tests/empty_result/notes.txt new file mode 100644 index 0000000..0a481f4 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/empty_result/notes.txt @@ -0,0 +1,26 @@ +This test requires Guard Duty. As this is a paid service, the test is disabled. + +The test can be enabled by running the following commands with a suitable profile or set of AWS credentials in play. + +1. Create the Guard Duty detector + + aws guardduty create-detector --enable + + +2. Get the detector ID + + aws guardduty list-detectors --query='DetectorIds[0]' + + +3. Copy the detector ID reported into terraform.tfvars and update the expected_variables.json file to match, replacing + 0123456789abcdef0123456789abcdef (unless that's your detector ID of course! ... It COULD happen!) + + +4. Change the RUN_TEST to true in ./test.sh + + +Once you've finished the testing, revert the changes above, and disable the detector using + + aws guardduty delete-detector --detector-id + +replacing with the detector ID you extracted in step 2 above. diff --git a/modules/util/terraform-aws-cli/tests/empty_result/terraform.tfvars b/modules/util/terraform-aws-cli/tests/empty_result/terraform.tfvars new file mode 100644 index 0000000..5e8c5e5 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/empty_result/terraform.tfvars @@ -0,0 +1,3 @@ +// An empty result from AWS +aws_cli_commands = ["guardduty", "update-detector", "--finding-publishing-frequency", "ONE_HOUR", "--detector-id", "0123456789abcdef0123456789abcdef"] +role_session_name = "empty_result" diff --git a/modules/util/terraform-aws-cli/tests/empty_result/test.sh b/modules/util/terraform-aws-cli/tests/empty_result/test.sh new file mode 100755 index 0000000..3098091 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/empty_result/test.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +function run_test() { +if [[ ! -f $PLAN_FILE ]]; then + echo "Failed to generate a plan - $PLAN_FILE"; + exit 1; +fi + +if [[ ! "$(terraform show -json $PLAN_FILE | jq -MSr .variables)" == "$(cat $EXPECTED_VARIABLES)" ]]; then + echo 'Failed to incorporate expected variable values into plan.'; + exit 2; +fi + +terraform apply -auto-approve -backup=- -state-out $STATE_FILE -var-file $TERRAFORM_TFVARS > $APPLY_LOG_FILE 2> $APPLY_ERROR_FILE + +if [[ ! -f $STATE_FILE ]]; then + echo "Failed to generate state file - $STATE_FILE"; + exit 3; +fi + +# Validate the presence of the plan error file. +if [[ ! -f $PLAN_ERROR_FILE ]]; then + echo "Failed to generate plan error file - $PLAN_ERROR_FILE"; + exit 4; +fi + +# Validate the plan error file is empty. +if [[ -s $PLAN_ERROR_FILE ]]; then + echo "Plan error file is not empty - $PLAN_ERROR_FILE"; + exit 5; +fi +} + +# Set to true to allow this test to run +RUN_TEST=false +if [[ "$RUN_TEST" == "false" ]]; then + echo "Start : $(dirname $0)"; + echo "Skipped : $(dirname $0) : See $(dirname $0)/notes.txt"; +else + . tests/common.sh $0 +fi diff --git a/modules/util/terraform-aws-cli/tests/role_session_name_invalid_characters/terraform.tfvars b/modules/util/terraform-aws-cli/tests/role_session_name_invalid_characters/terraform.tfvars new file mode 100644 index 0000000..4db5bf2 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/role_session_name_invalid_characters/terraform.tfvars @@ -0,0 +1,4 @@ +// 64 characters, but $ is invalid +role_session_name = "$234567890123456789012345678901234567890123456789012345678901234" +aws_cli_commands = ["version"] +debug_log_filename = "test-reports/role_session_name_invalid_characters/debug.log" diff --git a/modules/util/terraform-aws-cli/tests/role_session_name_invalid_characters/test.sh b/modules/util/terraform-aws-cli/tests/role_session_name_invalid_characters/test.sh new file mode 100755 index 0000000..e7a93c5 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/role_session_name_invalid_characters/test.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +function run_test() { +if [[ -f $PLAN_FILE ]]; then + echo "Incorrectly generated a plan - $PLAN_FILE"; + exit 1; +fi + +if [[ ! -z "$(cat $PLAN_LOG_FILE)" ]]; then + echo "Incorrectly generated content in the plan log file - $PLAN_LOG_FILE"; + exit 2; +fi + +if [[ ! "$(cat $PLAN_ERROR_FILE)" == *'The role session name match the regular expression'* ]]; then + echo 'Failed to detect invalid characters in role_session_name.'; + exit 3; +fi +} + +. tests/common.sh $0 diff --git a/modules/util/terraform-aws-cli/tests/role_session_name_optional/expected_variables.json b/modules/util/terraform-aws-cli/tests/role_session_name_optional/expected_variables.json new file mode 100644 index 0000000..aff5d95 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/role_session_name_optional/expected_variables.json @@ -0,0 +1,23 @@ +{ + "assume_role_arn": { + "value": "" + }, + "aws_cli_commands": { + "value": [ + "s3api", + "list-objects", + "--bucket", + "ryft-public-sample-data", + "--no-sign-request" + ] + }, + "aws_cli_query": { + "value": "max_by(Contents, &Size)" + }, + "debug_log_filename": { + "value": "" + }, + "role_session_name": { + "value": "" + } +} diff --git a/modules/util/terraform-aws-cli/tests/role_session_name_optional/terraform.tfvars b/modules/util/terraform-aws-cli/tests/role_session_name_optional/terraform.tfvars new file mode 100644 index 0000000..c50a0a0 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/role_session_name_optional/terraform.tfvars @@ -0,0 +1,3 @@ +// ryft-public-sample-data is a publicly accessible S3 bucket. +aws_cli_commands = ["s3api", "list-objects", "--bucket", "ryft-public-sample-data", "--no-sign-request"] +aws_cli_query = "max_by(Contents, &Size)" diff --git a/modules/util/terraform-aws-cli/tests/role_session_name_optional/test.sh b/modules/util/terraform-aws-cli/tests/role_session_name_optional/test.sh new file mode 100755 index 0000000..641a619 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/role_session_name_optional/test.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +function run_test() { +if [[ ! -f $PLAN_FILE ]]; then + echo "Failed to generate a plan - $PLAN_FILE"; + exit 1; +fi + +if [[ ! "$(terraform show -json $PLAN_FILE | jq -MSr .variables)" == "$(cat $EXPECTED_VARIABLES)" ]]; then + echo 'Failed to incorporate expected variable values into plan.'; + exit 2; +fi + +terraform apply -auto-approve -backup=- -state-out $STATE_FILE -var-file $TERRAFORM_TFVARS > $APPLY_LOG_FILE 2> $APPLY_ERROR_FILE + +if [[ ! -f $STATE_FILE ]]; then + echo "Failed to generate state file - $STATE_FILE"; + exit 3; +fi + +# Extract some content the state file. +if [[ ! "$(cat $STATE_FILE)" == *'0ae8f910a30bc83fd81c4e3c1a6bbd9bab0afe4e0762b56a2807d22fcd77d517'* ]]; then + echo 'Failed to retrieve expected content from AWS.'; + exit 4; +fi + +# Extract some content from the apply log. +if [[ ! "$(cat $APPLY_LOG_FILE)" == *"0ae8f910a30bc83fd81c4e3c1a6bbd9bab0afe4e0762b56a2807d22fcd77d517"* ]]; then + echo 'Failed to present expected content to Terraform.'; + exit 5; +fi + +# Validate the absence of the debug log. +if [[ -f $DEBUG_LOG_FILE ]]; then + echo "Incorrectly generated debug.log file - $DEBUG_LOG_FILE"; + exit 6; +fi +} + +. tests/common.sh $0 diff --git a/modules/util/terraform-aws-cli/tests/role_session_name_too_long/terraform.tfvars b/modules/util/terraform-aws-cli/tests/role_session_name_too_long/terraform.tfvars new file mode 100644 index 0000000..3e0e2b3 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/role_session_name_too_long/terraform.tfvars @@ -0,0 +1,4 @@ +// 65 characters is too long +role_session_name = "12345678901234567890123456789012345678901234567890123456789012345" +aws_cli_commands = ["version"] +debug_log_filename = "test-reports/role_session_name_too_long/debug.log" diff --git a/modules/util/terraform-aws-cli/tests/role_session_name_too_long/test.sh b/modules/util/terraform-aws-cli/tests/role_session_name_too_long/test.sh new file mode 100755 index 0000000..f755c29 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/role_session_name_too_long/test.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +function run_test() { +if [[ -f $PLAN_FILE ]]; then + echo "Incorrectly generated a plan - $PLAN_FILE"; + exit 1; +fi + +if [[ ! -z "$(cat $PLAN_LOG_FILE)" ]]; then + echo "Incorrectly generated content in the plan log file - $PLAN_LOG_FILE"; + exit 2; +fi + +if [[ ! "$(cat $PLAN_ERROR_FILE)" == *'The role session name must be less than or equal to 64 characters'* ]]; then + echo 'Failed to detect too long role_session_name.'; + exit 3; +fi +} + +. tests/common.sh $0 diff --git a/modules/util/terraform-aws-cli/tests/test_with_debug/expected_variables.json b/modules/util/terraform-aws-cli/tests/test_with_debug/expected_variables.json new file mode 100644 index 0000000..62a36aa --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/test_with_debug/expected_variables.json @@ -0,0 +1,23 @@ +{ + "assume_role_arn": { + "value": "" + }, + "aws_cli_commands": { + "value": [ + "s3api", + "list-objects", + "--bucket", + "ryft-public-sample-data", + "--no-sign-request" + ] + }, + "aws_cli_query": { + "value": "max_by(Contents, &Size)" + }, + "debug_log_filename": { + "value": "test-reports/test_with_debug/debug.log" + }, + "role_session_name": { + "value": "test_with_debug" + } +} diff --git a/modules/util/terraform-aws-cli/tests/test_with_debug/terraform.tfvars b/modules/util/terraform-aws-cli/tests/test_with_debug/terraform.tfvars new file mode 100644 index 0000000..6292ed1 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/test_with_debug/terraform.tfvars @@ -0,0 +1,5 @@ +// ryft-public-sample-data is a publicly accessible S3 bucket. +aws_cli_commands = ["s3api", "list-objects", "--bucket", "ryft-public-sample-data", "--no-sign-request"] +aws_cli_query = "max_by(Contents, &Size)" +debug_log_filename = "test-reports/test_with_debug/debug.log" +role_session_name = "test_with_debug" diff --git a/modules/util/terraform-aws-cli/tests/test_with_debug/test.sh b/modules/util/terraform-aws-cli/tests/test_with_debug/test.sh new file mode 100755 index 0000000..74b36e9 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/test_with_debug/test.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +function run_test() { +if [[ ! -f $PLAN_FILE ]]; then + echo "Failed to generate a plan - $PLAN_FILE"; + exit 1; +fi + +if [[ ! "$(terraform show -json $PLAN_FILE | jq -MSr .variables)" == "$(cat $EXPECTED_VARIABLES)" ]]; then + echo 'Failed to incorporate expected variable values into plan.'; + exit 2; +fi + +terraform apply -auto-approve -backup=- -state-out $STATE_FILE -var-file $TERRAFORM_TFVARS > $APPLY_LOG_FILE 2> $APPLY_ERROR_FILE + +if [[ ! -f $STATE_FILE ]]; then + echo "Failed to generate state file - $STATE_FILE"; + exit 3; +fi + +# Extract some content the state file. +if [[ ! "$(cat $STATE_FILE)" == *'0ae8f910a30bc83fd81c4e3c1a6bbd9bab0afe4e0762b56a2807d22fcd77d517'* ]]; then + echo 'Failed to retrieve expected content from AWS.'; + exit 4; +fi + +# Extract some content from the apply log. +if [[ ! "$(cat $APPLY_LOG_FILE)" == *"0ae8f910a30bc83fd81c4e3c1a6bbd9bab0afe4e0762b56a2807d22fcd77d517"* ]]; then + echo 'Failed to present expected content to Terraform.'; + exit 5; +fi + +# Validate the presence of the debug log. +if [[ ! -f $DEBUG_LOG_FILE ]]; then + echo "Failed to generate debug.log file - $DEBUG_LOG_FILE"; + exit 6; +fi +} + +. tests/common.sh $0 diff --git a/modules/util/terraform-aws-cli/tests/test_without_debug/expected_variables.json b/modules/util/terraform-aws-cli/tests/test_without_debug/expected_variables.json new file mode 100644 index 0000000..984aa62 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/test_without_debug/expected_variables.json @@ -0,0 +1,23 @@ +{ + "assume_role_arn": { + "value": "" + }, + "aws_cli_commands": { + "value": [ + "s3api", + "list-objects", + "--bucket", + "ryft-public-sample-data", + "--no-sign-request" + ] + }, + "aws_cli_query": { + "value": "max_by(Contents, &Size)" + }, + "debug_log_filename": { + "value": "" + }, + "role_session_name": { + "value": "test_without_debug" + } +} diff --git a/modules/util/terraform-aws-cli/tests/test_without_debug/terraform.tfvars b/modules/util/terraform-aws-cli/tests/test_without_debug/terraform.tfvars new file mode 100644 index 0000000..f7f2c31 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/test_without_debug/terraform.tfvars @@ -0,0 +1,4 @@ +// ryft-public-sample-data is a publicly accessible S3 bucket. +aws_cli_commands = ["s3api", "list-objects", "--bucket", "ryft-public-sample-data", "--no-sign-request"] +aws_cli_query = "max_by(Contents, &Size)" +role_session_name = "test_without_debug" diff --git a/modules/util/terraform-aws-cli/tests/test_without_debug/test.sh b/modules/util/terraform-aws-cli/tests/test_without_debug/test.sh new file mode 100755 index 0000000..641a619 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/test_without_debug/test.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +function run_test() { +if [[ ! -f $PLAN_FILE ]]; then + echo "Failed to generate a plan - $PLAN_FILE"; + exit 1; +fi + +if [[ ! "$(terraform show -json $PLAN_FILE | jq -MSr .variables)" == "$(cat $EXPECTED_VARIABLES)" ]]; then + echo 'Failed to incorporate expected variable values into plan.'; + exit 2; +fi + +terraform apply -auto-approve -backup=- -state-out $STATE_FILE -var-file $TERRAFORM_TFVARS > $APPLY_LOG_FILE 2> $APPLY_ERROR_FILE + +if [[ ! -f $STATE_FILE ]]; then + echo "Failed to generate state file - $STATE_FILE"; + exit 3; +fi + +# Extract some content the state file. +if [[ ! "$(cat $STATE_FILE)" == *'0ae8f910a30bc83fd81c4e3c1a6bbd9bab0afe4e0762b56a2807d22fcd77d517'* ]]; then + echo 'Failed to retrieve expected content from AWS.'; + exit 4; +fi + +# Extract some content from the apply log. +if [[ ! "$(cat $APPLY_LOG_FILE)" == *"0ae8f910a30bc83fd81c4e3c1a6bbd9bab0afe4e0762b56a2807d22fcd77d517"* ]]; then + echo 'Failed to present expected content to Terraform.'; + exit 5; +fi + +# Validate the absence of the debug log. +if [[ -f $DEBUG_LOG_FILE ]]; then + echo "Incorrectly generated debug.log file - $DEBUG_LOG_FILE"; + exit 6; +fi +} + +. tests/common.sh $0 diff --git a/modules/util/terraform-aws-cli/tests/tests.sh b/modules/util/terraform-aws-cli/tests/tests.sh new file mode 100755 index 0000000..a315cd0 --- /dev/null +++ b/modules/util/terraform-aws-cli/tests/tests.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash -e +rm -rf temp +rm -rf test-reports +find . -type f -name test.sh | sort | xargs -L 1 bash diff --git a/modules/util/terraform-aws-cli/variables.tf b/modules/util/terraform-aws-cli/variables.tf new file mode 100644 index 0000000..52d2fc1 --- /dev/null +++ b/modules/util/terraform-aws-cli/variables.tf @@ -0,0 +1,43 @@ +variable "assume_role_arn" { + description = "The ARN of the role being assumed (optional)" + type = string + default = "" + + validation { + condition = can(regex("^(?:arn:aws(?:-cn|-us-gov|):(?:iam|sts)::[0-9]{12}:.+|)$", var.assume_role_arn)) + error_message = "The optional ARN must match the format documented in https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html." + } +} + +variable "aws_cli_commands" { + description = "The AWS CLI command and subcommands" + type = list(string) +} + +variable "aws_cli_query" { + description = "The --query value" + type = string + default = "" +} + +variable "role_session_name" { + description = "The role session name" + type = string + default = "" + + validation { + condition = length(var.role_session_name) <= 64 + error_message = "The role session name must be less than or equal to 64 characters." + } + + validation { + condition = can(regex("^[\\w+=,.@-]*$", var.role_session_name)) + error_message = "The role session name match the regular expression '^[\\w+=,.@-]*$'." + } +} + +variable "debug_log_filename" { + description = "Generate a debug log if a `debug_log_filename` is supplied" + type = string + default = "" +} diff --git a/modules/util/terraform-aws-cli/versions.tf b/modules/util/terraform-aws-cli/versions.tf new file mode 100644 index 0000000..df95b58 --- /dev/null +++ b/modules/util/terraform-aws-cli/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 0.15" + required_providers { + external = { + source = "hashicorp/external" + version = "~> 2.0" + } + local = { + source = "hashicorp/local" + version = "~> 2.0" + } + } +}