amazon-web-services - Amazon EMR 使用 Spot 实例作为核心，使用 Terraform 的主节点

Question

我想使用 terraform 将 Spot 实例用于 EMR。我可以使用 CLI 做到这一点：

aws emr create-cluster --release-label emr-5.3.1 --service-role 
EMR_DefaultRole \
--ec2-attributes InstanceProfile=EMR_EC2_DefaultRole \
--instance-fleets 
InstanceFleetType=MASTER,TargetSpotCapacity=1,InstanceTypeConfigs= . 
['{InstanceType=m4.large,BidPrice=0.5}'] \
InstanceFleetType=CORE,TargetSpotCapacity=1,InstanceTypeConfigs= . 
['{InstanceType=m4.large,BidPrice=0.5}']`

但是我在 terraform 中找不到实例队列功能。我如何使用 terraform 完成相同的任务

标签： amazon-web-servicesterraformamazon-emr

Answer 1

instance_group您可以为 EMR 集群在块下使用的实例指定出价，以使 Terraform 为这些实例使用 Spot 实例。

资源文档中的示例aws_emr_cluster显示了如何指定它：

resource "aws_emr_cluster" "emr-test-cluster" {
  name          = "emr-test-arn"
  release_label = "emr-4.6.0"
  applications  = ["Spark"]
  additional_info = <<EOF
{
  "instanceAwsClientConfiguration": {
    "proxyPort": 8099,
    "proxyHost": "myproxy.example.com"
  }
}
EOF

  termination_protection = false
  keep_job_flow_alive_when_no_steps = true

  ec2_attributes {
    subnet_id                         = "${aws_subnet.main.id}"
    emr_managed_master_security_group = "${aws_security_group.sg.id}"
    emr_managed_slave_security_group  = "${aws_security_group.sg.id}"
    instance_profile                  = "${aws_iam_instance_profile.emr_profile.arn}"
  }

  instance_group {
      instance_role = "CORE"
      instance_type = "c4.large"
      instance_count = "1"
      ebs_config {
        size = "40"
        type = "gp2"
        volumes_per_instance = 1
      }
      bid_price = "0.30"
      autoscaling_policy = <<EOF
{
"Constraints": {
  "MinCapacity": 1,
  "MaxCapacity": 2
},
"Rules": [
  {
    "Name": "ScaleOutMemoryPercentage",
    "Description": "Scale out if YARNMemoryAvailablePercentage is less than 15",
    "Action": {
      "SimpleScalingPolicyConfiguration": {
        "AdjustmentType": "CHANGE_IN_CAPACITY",
        "ScalingAdjustment": 1,
        "CoolDown": 300
      }
    },
    "Trigger": {
      "CloudWatchAlarmDefinition": {
        "ComparisonOperator": "LESS_THAN",
        "EvaluationPeriods": 1,
        "MetricName": "YARNMemoryAvailablePercentage",
        "Namespace": "AWS/ElasticMapReduce",
        "Period": 300,
        "Statistic": "AVERAGE",
        "Threshold": 15.0,
        "Unit": "PERCENT"
      }
    }
  }
]
}
EOF
}
  ebs_root_volume_size     = 100

  master_instance_type = "m3.xlarge"
  core_instance_type   = "m3.xlarge"
  core_instance_count  = 1

  tags {
    role     = "rolename"
    env      = "env"
  }

  bootstrap_action {
    path = "s3://elasticmapreduce/bootstrap-actions/run-if"
    name = "runif"
    args = ["instance.isMaster=true", "echo running on master node"]
  }

  configurations_json = <<EOF
  [
    {
      "Classification": "hadoop-env",
      "Configurations": [
        {
          "Classification": "export",
          "Properties": {
            "JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
          }
        }
      ],
      "Properties": {}
    },
    {
      "Classification": "spark-env",
      "Configurations": [
        {
          "Classification": "export",
          "Properties": {
            "JAVA_HOME": "/usr/lib/jvm/java-1.8.0"
          }
        }
      ],
      "Properties": {}
    }
  ]
EOF  

  service_role = "${aws_iam_role.iam_emr_service_role.arn}"
}