Skip to content

Commit 926bde2

Browse files
baibailihabaibailiha
authored
fix: Correct issue where custom launch template is not used when EKS managed node group is used externally (#1824)
1 parent 0cce5e2 commit 926bde2

File tree

5 files changed

+112
-16
lines changed

5 files changed

+112
-16
lines changed

‎examples/eks_managed_node_group/README.md

+30
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,33 @@ Configuration in this directory creates an AWS EKS cluster with various EKS Mana
1212

1313
See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) for further details.
1414

15+
## Container Runtime & User Data
16+
17+
When using the default AMI provided by the EKS Managed Node Group service (i.e. - not specifying a value for `ami_id`), users should be aware of the limitations of configuring the node bootstrap process via user data. Due to not having direct access to the bootrap.sh script invocation and therefore its configuration flags (this is provide by the EKS Managed Node Group service in the node user data), a work around for ensuring the appropriate configuration settings is shown below. The following example shows how to inject configuration variables ahead of the merged user data provided by the EKS Managed Node Group service as well as how to enable the containerd runtime using this approach. More details can be found [here](https://github.com/awslabs/amazon-eks-ami/issues/844).
18+
19+
```hcl
20+
...
21+
# Demo of containerd usage when not specifying a custom AMI ID
22+
# (merged into user data before EKS MNG provided user data)
23+
containerd = {
24+
name = "containerd"
25+
26+
# See issue https://github.com/awslabs/amazon-eks-ami/issues/844
27+
pre_bootstrap_user_data = <<-EOT
28+
#!/bin/bash
29+
set -ex
30+
cat <<-EOF > /etc/profile.d/bootstrap.sh
31+
export CONTAINER_RUNTIME="containerd"
32+
export USE_MAX_PODS=false
33+
export KUBELET_EXTRA_ARGS="--max-pods=110"
34+
EOF
35+
# Source extra environment variables in bootstrap script
36+
sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh
37+
EOT
38+
}
39+
...
40+
```
41+
1542
## Usage
1643

1744
To run this example you need to execute:
@@ -63,6 +90,9 @@ Note that this example may create resources which cost money. Run `terraform des
6390
| [aws_security_group.remote_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource |
6491
| [null_resource.patch](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
6592
| [tls_private_key.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource |
93+
| [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
94+
| [aws_ami.eks_default_arm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
95+
| [aws_ami.eks_default_bottlerocket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
6696
| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
6797
| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source |
6898
| [aws_iam_policy_document.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |

‎examples/eks_managed_node_group/main.tf

+56-7
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ module "eks" {
133133
# Custom AMI, using module provided bootstrap data
134134
bottlerocket_custom = {
135135
# Current bottlerocket AMI
136-
ami_id = "ami-0ff61e0bcfc81dc94"
136+
ami_id = data.aws_ami.eks_default_bottlerocket.image_id
137137
platform = "bottlerocket"
138138

139139
# use module user data template to boostrap
@@ -165,7 +165,7 @@ module "eks" {
165165
custom_ami = {
166166
ami_type = "AL2_ARM_64"
167167
# Current default AMI used by managed node groups - pseudo "custom"
168-
ami_id = "ami-01dc0aa438e3214c2" # ARM
168+
ami_id = data.aws_ami.eks_default_arm.image_id
169169

170170
# This will ensure the boostrap user data is used to join the node
171171
# By default, EKS managed node groups will not append bootstrap script;
@@ -176,6 +176,25 @@ module "eks" {
176176
instance_types = ["t4g.medium"]
177177
}
178178

179+
# Demo of containerd usage when not specifying a custom AMI ID
180+
# (merged into user data before EKS MNG provided user data)
181+
containerd = {
182+
name = "containerd"
183+
184+
# See issue https://github.com/awslabs/amazon-eks-ami/issues/844
185+
pre_bootstrap_user_data = <<-EOT
186+
#!/bin/bash
187+
set -ex
188+
cat <<-EOF > /etc/profile.d/bootstrap.sh
189+
export CONTAINER_RUNTIME="containerd"
190+
export USE_MAX_PODS=false
191+
export KUBELET_EXTRA_ARGS="--max-pods=110"
192+
EOF
193+
# Source extra environment variables in bootstrap script
194+
sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh
195+
EOT
196+
}
197+
179198
# Complete
180199
complete = {
181200
name = "complete-eks-mng"
@@ -187,23 +206,23 @@ module "eks" {
187206
max_size = 7
188207
desired_size = 1
189208

190-
ami_id = "ami-0caf35bc73450c396"
209+
ami_id = data.aws_ami.eks_default.image_id
191210
enable_bootstrap_user_data = true
192211
bootstrap_extra_args = "--container-runtime containerd --kubelet-extra-args '--max-pods=20'"
193212

194213
pre_bootstrap_user_data = <<-EOT
195-
export CONTAINER_RUNTIME="containerd"
196-
export USE_MAX_PODS=false
214+
export CONTAINER_RUNTIME="containerd"
215+
export USE_MAX_PODS=false
197216
EOT
198217

199218
post_bootstrap_user_data = <<-EOT
200-
echo "you are free little kubelet!"
219+
echo "you are free little kubelet!"
201220
EOT
202221

203222
capacity_type = "SPOT"
204223
disk_size = 256
205224
force_update_version = true
206-
instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large", "m3.large", "m4.large"]
225+
instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"]
207226
labels = {
208227
GithubRepo = "terraform-aws-eks"
209228
GithubOrg = "terraform-aws-modules"
@@ -619,3 +638,33 @@ resource "aws_iam_policy" "node_additional" {
619638

620639
tags = local.tags
621640
}
641+
642+
data "aws_ami" "eks_default" {
643+
most_recent = true
644+
owners = ["amazon"]
645+
646+
filter {
647+
name = "name"
648+
values = ["amazon-eks-node-${local.cluster_version}-v*"]
649+
}
650+
}
651+
652+
data "aws_ami" "eks_default_arm" {
653+
most_recent = true
654+
owners = ["amazon"]
655+
656+
filter {
657+
name = "name"
658+
values = ["amazon-eks-arm64-node-${local.cluster_version}-v*"]
659+
}
660+
}
661+
662+
data "aws_ami" "eks_default_bottlerocket" {
663+
most_recent = true
664+
owners = ["amazon"]
665+
666+
filter {
667+
name = "name"
668+
values = ["bottlerocket-aws-k8s-${local.cluster_version}-x86_64-*"]
669+
}
670+
}

‎examples/self_managed_node_group/README.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ Note that this example may create resources which cost money. Run `terraform des
5555
| [aws_security_group.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource |
5656
| [null_resource.apply](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
5757
| [tls_private_key.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource |
58-
| [aws_ami.bottlerocket_ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
58+
| [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
59+
| [aws_ami.eks_default_bottlerocket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
5960
| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
6061
| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source |
6162
| [aws_iam_policy_document.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |

‎examples/self_managed_node_group/main.tf

+16-6
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ module "eks" {
9494
name = "bottlerocket-self-mng"
9595

9696
platform = "bottlerocket"
97-
ami_id = data.aws_ami.bottlerocket_ami.id
97+
ami_id = data.aws_ami.eks_default_bottlerocket.id
9898
instance_type = "m5.large"
9999
desired_size = 2
100100
key_name = aws_key_pair.this.key_name
@@ -159,16 +159,16 @@ module "eks" {
159159
max_size = 7
160160
desired_size = 1
161161

162-
ami_id = "ami-0caf35bc73450c396"
162+
ami_id = data.aws_ami.eks_default.id
163163
bootstrap_extra_args = "--kubelet-extra-args '--max-pods=110'"
164164

165165
pre_bootstrap_user_data = <<-EOT
166-
export CONTAINER_RUNTIME="containerd"
167-
export USE_MAX_PODS=false
166+
export CONTAINER_RUNTIME="containerd"
167+
export USE_MAX_PODS=false
168168
EOT
169169

170170
post_bootstrap_user_data = <<-EOT
171-
echo "you are free little kubelet!"
171+
echo "you are free little kubelet!"
172172
EOT
173173

174174
disk_size = 256
@@ -374,7 +374,17 @@ resource "aws_kms_key" "eks" {
374374
tags = local.tags
375375
}
376376

377-
data "aws_ami" "bottlerocket_ami" {
377+
data "aws_ami" "eks_default" {
378+
most_recent = true
379+
owners = ["amazon"]
380+
381+
filter {
382+
name = "name"
383+
values = ["amazon-eks-node-${local.cluster_version}-v*"]
384+
}
385+
}
386+
387+
data "aws_ami" "eks_default_bottlerocket" {
378388
most_recent = true
379389
owners = ["amazon"]
380390

‎modules/eks-managed-node-group/main.tf

+8-2
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,14 @@ module "user_data" {
3030
################################################################################
3131

3232
locals {
33-
use_custom_launch_template = var.launch_template_name != ""
34-
launch_template_name_int = coalesce(var.launch_template_name, "${var.name}-eks-node-group")
33+
# There are 4 scenarios here that have to be considered for `use_custom_launch_template`:
34+
# 1. `var.create_launch_template = false && var.launch_template_name == ""` => EKS MNG will use its own default LT
35+
# 2. `var.create_launch_template = false && var.launch_template_name == "something"` => User provided custom LT will be used
36+
# 3. `var.create_launch_template = true && var.launch_template_name == ""` => Custom LT will be used, module will provide a default name
37+
# 4. `var.create_launch_template = true && var.launch_template_name == "something"` => Custom LT will be used, LT name is provided by user
38+
use_custom_launch_template = var.create_launch_template || var.launch_template_name != ""
39+
40+
launch_template_name_int = coalesce(var.launch_template_name, "${var.name}-eks-node-group")
3541
}
3642

3743
resource "aws_launch_template" "this" {

0 commit comments

Comments
 (0)