DEL: moving EMR to examples
This commit is contained in:
parent
3ac1e09361
commit
f3cd4b79bb
GPG Key ID:
CD4FF6793F09AB86
@ -1,197 +0,0 @@
|
||||
locals {
|
||||
name = "${var.environment}-${var.customer-name}"
|
||||
}
|
||||
|
||||
module "emr" {
|
||||
source = "terraform-aws-modules/emr/aws"
|
||||
version = "1.2.0"
|
||||
|
||||
name = "${local.name}-emr"
|
||||
release_label = "emr-7.0.0"
|
||||
security_configuration_name = aws_emr_security_configuration.security_config.name
|
||||
applications = ["hbase", "phoenix"]
|
||||
auto_termination_policy = {
|
||||
idle_timeout = 3600
|
||||
}
|
||||
|
||||
bootstrap_action = {
|
||||
}
|
||||
|
||||
configurations_json = jsonencode([
|
||||
{
|
||||
Classification : "hbase-env",
|
||||
Configurations : [
|
||||
{
|
||||
"Classification" : "export",
|
||||
"Properties" : {
|
||||
"HBASE_MASTER_OPTS" : "-Xmx4g",
|
||||
"HBASE_REGIONSERVER_OPTS" : "-Xmx8g"
|
||||
}
|
||||
}
|
||||
],
|
||||
Properties : {}
|
||||
},
|
||||
{
|
||||
Classification : "hbase-site",
|
||||
Properties : {
|
||||
"hbase.regionserver.handler.count" : "300"
|
||||
}
|
||||
}
|
||||
])
|
||||
|
||||
master_instance_fleet = {
|
||||
name = "master-fleet"
|
||||
target_on_demand_capacity = 1
|
||||
instance_type_configs = [
|
||||
{
|
||||
instance_type = "c6g.xlarge"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
core_instance_fleet = {
|
||||
name = "core-fleet"
|
||||
target_on_demand_capacity = 0
|
||||
target_spot_capacity = 2
|
||||
instance_type_configs = [
|
||||
{
|
||||
bid_price_as_percentage_of_on_demand_price = 70
|
||||
instance_type = "c6g.xlarge"
|
||||
weighted_capacity = 1
|
||||
ebs_config = {
|
||||
size = 20
|
||||
type = "gp3"
|
||||
volumes_per_instance = 1
|
||||
}
|
||||
},
|
||||
{
|
||||
bid_price_as_percentage_of_on_demand_price = 70
|
||||
instance_type = "m6g.xlarge"
|
||||
weighted_capacity = 1
|
||||
ebs_config = {
|
||||
size = 20
|
||||
type = "gp3"
|
||||
volumes_per_instance = 1
|
||||
}
|
||||
}
|
||||
]
|
||||
launch_specifications = {
|
||||
spot_specification = {
|
||||
allocation_strategy = "capacity-optimized"
|
||||
block_duration_minutes = 0
|
||||
timeout_action = "SWITCH_TO_ON_DEMAND"
|
||||
timeout_duration_minutes = 5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ebs_root_volume_size = 20
|
||||
# Subnets should be tagged with
|
||||
# { "for-use-with-amazon-emr-managed-policies" = true }
|
||||
ec2_attributes = {
|
||||
subnet_ids = ["subnet-08dec6787782ee087", "subnet-0551e96ffd016192a"]
|
||||
key_name = "kf-key"
|
||||
}
|
||||
vpc_id = "vpc-01a10b033169f89a8"
|
||||
|
||||
# Required for creating public cluster
|
||||
is_private_cluster = false
|
||||
|
||||
keep_job_flow_alive_when_no_steps = true
|
||||
list_steps_states = ["PENDING", "RUNNING", "CANCEL_PENDING", "CANCELLED", "FAILED", "INTERRUPTED", "COMPLETED"]
|
||||
log_uri = "s3n://${module.s3_bucket.s3_bucket_id}/"
|
||||
|
||||
scale_down_behavior = "TERMINATE_AT_TASK_COMPLETION"
|
||||
step_concurrency_level = 3
|
||||
termination_protection = false
|
||||
visible_to_all_users = true
|
||||
service_iam_role_policies = {
|
||||
AmazonEMRServicePolicy_v2 = "arn:aws:iam::aws:policy/service-role/AmazonEMRServicePolicy_v2"
|
||||
PowerUser = "arn:aws:iam::aws:policy/PowerUserAccess"
|
||||
}
|
||||
}
|
||||
|
||||
resource "random_id" "this" {
|
||||
byte_length = 2
|
||||
}
|
||||
|
||||
module "s3_bucket" {
|
||||
source = "terraform-aws-modules/s3-bucket/aws"
|
||||
version = "~> 3.0"
|
||||
|
||||
bucket = "${local.name}-emrlogs-${random_id.this.dec}"
|
||||
|
||||
# Allow deletion of non-empty bucket
|
||||
# Example usage only - not recommended for production
|
||||
force_destroy = true
|
||||
|
||||
attach_deny_insecure_transport_policy = true
|
||||
attach_require_latest_tls_policy = true
|
||||
|
||||
block_public_acls = true
|
||||
block_public_policy = true
|
||||
ignore_public_acls = true
|
||||
restrict_public_buckets = true
|
||||
|
||||
server_side_encryption_configuration = {
|
||||
rule = {
|
||||
apply_server_side_encryption_by_default = {
|
||||
sse_algorithm = "AES256"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
resource "aws_emr_security_configuration" "security_config" {
|
||||
name = "${local.name}-emr-security-config"
|
||||
|
||||
configuration = jsonencode(
|
||||
{
|
||||
"EncryptionConfiguration" : {
|
||||
"AtRestEncryptionConfiguration" : {
|
||||
"S3EncryptionConfiguration" : {
|
||||
"EncryptionMode" : "SSE-S3"
|
||||
},
|
||||
},
|
||||
"EnableInTransitEncryption" : false,
|
||||
"EnableAtRestEncryption" : true
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
# Tag EMR master and core instances
|
||||
# Need to run this layer twice to set instance tags
|
||||
# Adding depends_on will results in dependency loop
|
||||
data "aws_instances" "master_instances" {
|
||||
# depends_on = [module.emr]
|
||||
instance_tags = {
|
||||
"aws:elasticmapreduce:instance-group-role" = "MASTER"
|
||||
}
|
||||
instance_state_names = ["running"]
|
||||
}
|
||||
|
||||
data "aws_instances" "core_instances" {
|
||||
# depends_on = [module.emr]
|
||||
instance_tags = {
|
||||
"aws:elasticmapreduce:instance-group-role" = "CORE"
|
||||
}
|
||||
instance_state_names = ["running"]
|
||||
}
|
||||
|
||||
resource "aws_ec2_tag" "tag-emr-core-instances" {
|
||||
# depends_on = [data.aws_instances.core_instances]
|
||||
count = length(data.aws_instances.core_instances.ids)
|
||||
resource_id = sort(data.aws_instances.core_instances.ids)[count.index]
|
||||
key = "Name"
|
||||
value = "${local.name}-emr-core-${count.index + 1}"
|
||||
}
|
||||
|
||||
resource "aws_ec2_tag" "tag-emr-master-instances" {
|
||||
# depends_on = [data.aws_instances.master_instances]
|
||||
count = length(data.aws_instances.master_instances.ids)
|
||||
resource_id = sort(data.aws_instances.master_instances.ids)[count.index]
|
||||
key = "Name"
|
||||
value = "${local.name}-emr-master-${count.index + 1}"
|
||||
}
|
||||
@ -1,7 +0,0 @@
|
||||
output "core_instance_ids" {
|
||||
value = data.aws_instances.core_instances.ids
|
||||
}
|
||||
|
||||
output "master_instance_ids" {
|
||||
value = data.aws_instances.master_instances.ids
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
provider "aws" {
|
||||
region = var.aws-region
|
||||
default_tags {
|
||||
tags = {
|
||||
ServiceProvider = "RackspaceTechnology"
|
||||
Environment = var.environment
|
||||
Project = var.project
|
||||
Application = var.application
|
||||
TerraformMode = "managed"
|
||||
TerraformDir = "${reverse(split("/", path.cwd))[1]}/${reverse(split("/", path.cwd))[0]}"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
terraform {
|
||||
required_version = ">= 1.3.0"
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 5.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,6 +0,0 @@
|
||||
aws-region = "ap-east-1"
|
||||
# aws-region-short = "ape1"
|
||||
customer-name = "ken2026"
|
||||
environment = "lab"
|
||||
project = "iac"
|
||||
application = "emr"
|
||||
@ -1,9 +0,0 @@
|
||||
variable "aws-region" {}
|
||||
# variable "aws-region-short" {}
|
||||
variable "customer-name" {}
|
||||
variable "environment" {}
|
||||
variable "project" {}
|
||||
variable "application" {}
|
||||
locals {
|
||||
resource-prefix = "${var.environment}-${substr(var.aws-region, 0, 2)}-${var.customer-name}-${var.project}"
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user