fix: Updates from testing with Outposts, still need final remote validation hence the git push

Author: bryantbiggs

examples/outposts/README.md

@@ -10,10 +10,25 @@ Note: This example requires an an AWS Outpost to provision.
 
 To run this example you need to:
 
-1. Copy the `terraform.tfvars.example` to `terraform.tfvars` and fill in the required variables
-2. Execute:
+1. Deploy the remote host where the cluster will be provisioned from. The remote host is required since only private access is permitted to clusters created on Outposts. If you have access to the network where Outposts are provisioned (VPN, etc.), you can skip this step:
 
 ```bash
+$ cd prerequisites
+$ terraform init
+$ terraform plan
+$ terraform apply
+```
+
+2. If provisioning using the remote host deployed in step 1, connect to the remote host using SSM. Note, you will need to have the [SSM plugin for the AWS CLI installed](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-working-with-install-plugin.html). You can use the output generated by step 1 to connect:
+
+```bash
+$ aws ssm start-session --region <REGION> --target <INSTANCE_ID>
+```
+
+3. Once connected to the remote host, navigate to the cloned project example directory and deploy the example:
+
+```bash
+$ cd $HOME/terraform-aws-eks/examples/outposts
 $ terraform init
 $ terraform plan
 $ terraform apply
@@ -35,27 +50,31 @@ Note that this example may create resources which cost money. Run `terraform des
 | Name | Version |
 |------|---------|
 | <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.34 |
+| <a name="provider_kubernetes"></a> [kubernetes](#provider\_kubernetes) | >= 2.10 |
 
 ## Modules
 
 | Name | Source | Version |
 |------|--------|---------|
 | <a name="module_eks"></a> [eks](#module\_eks) | ../.. | n/a |
-| <a name="module_vpc_cni_irsa"></a> [vpc\_cni\_irsa](#module\_vpc\_cni\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.0 |
 
 ## Resources
 
 | Name | Type |
 |------|------|
+| [kubernetes_storage_class_v1.this](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/storage_class_v1) | resource |
+| [aws_outposts_outpost_instance_types.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/outposts_outpost_instance_types) | data source |
 | [aws_outposts_outposts.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/outposts_outposts) | data source |
+| [aws_subnet.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source |
+| [aws_subnets.lookup](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source |
 | [aws_subnets.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source |
+| [aws_vpc.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_outpost_instance_type"></a> [outpost\_instance\_type](#input\_outpost\_instance\_type) | Instance type supported by the Outposts instance | `string` | `"m5.large"` | no |
-| <a name="input_region"></a> [region](#input\_region) | The AWS region to deploy into (e.g. us-east-1) | `string` | n/a | yes |
+| <a name="input_region"></a> [region](#input\_region) | The AWS region to deploy into (e.g. us-east-1) | `string` | `"us-west-2"` | no |
 
 ## Outputs
 
@@ -64,32 +83,21 @@ Note that this example may create resources which cost money. Run `terraform des
 | <a name="output_aws_auth_configmap_yaml"></a> [aws\_auth\_configmap\_yaml](#output\_aws\_auth\_configmap\_yaml) | Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles |
 | <a name="output_cloudwatch_log_group_arn"></a> [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created |
 | <a name="output_cloudwatch_log_group_name"></a> [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created |
-| <a name="output_cluster_addons"></a> [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled |
 | <a name="output_cluster_arn"></a> [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster |
 | <a name="output_cluster_certificate_authority_data"></a> [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster |
 | <a name="output_cluster_endpoint"></a> [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server |
 | <a name="output_cluster_iam_role_arn"></a> [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster |
 | <a name="output_cluster_iam_role_name"></a> [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster |
 | <a name="output_cluster_iam_role_unique_id"></a> [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role |
 | <a name="output_cluster_id"></a> [cluster\_id](#output\_cluster\_id) | The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready |
-| <a name="output_cluster_identity_providers"></a> [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled |
 | <a name="output_cluster_oidc_issuer_url"></a> [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider |
 | <a name="output_cluster_platform_version"></a> [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster |
 | <a name="output_cluster_primary_security_group_id"></a> [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console |
 | <a name="output_cluster_security_group_arn"></a> [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group |
 | <a name="output_cluster_security_group_id"></a> [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group |
 | <a name="output_cluster_status"></a> [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` |
-| <a name="output_cluster_tls_certificate_sha1_fingerprint"></a> [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate |
-| <a name="output_eks_managed_node_groups"></a> [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created |
-| <a name="output_eks_managed_node_groups_autoscaling_group_names"></a> [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups |
-| <a name="output_fargate_profiles"></a> [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created |
-| <a name="output_kms_key_arn"></a> [kms\_key\_arn](#output\_kms\_key\_arn) | The Amazon Resource Name (ARN) of the key |
-| <a name="output_kms_key_id"></a> [kms\_key\_id](#output\_kms\_key\_id) | The globally unique identifier for the key |
-| <a name="output_kms_key_policy"></a> [kms\_key\_policy](#output\_kms\_key\_policy) | The IAM resource policy set on the key |
 | <a name="output_node_security_group_arn"></a> [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group |
 | <a name="output_node_security_group_id"></a> [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group |
-| <a name="output_oidc_provider"></a> [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) |
-| <a name="output_oidc_provider_arn"></a> [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` |
 | <a name="output_self_managed_node_groups"></a> [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created |
 | <a name="output_self_managed_node_groups_autoscaling_group_names"></a> [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/outposts/main.tf

@@ -10,14 +10,18 @@ provider "kubernetes" {
     api_version = "client.authentication.k8s.io/v1beta1"
     command     = "aws"
     # This requires the awscli to be installed locally where Terraform is executed
-    args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id]
+    # Note: The profile is set in the user-data of `prerequisites` to use the EC2 metadata for credentials
+    args = ["eks", "get-token", "--cluster-name", module.eks.cluster_id, "--region", var.region, "--profile", "terraform"]
   }
 }
 
 locals {
   name            = "ex-${basename(path.cwd)}"
   cluster_version = "1.21" # Required by EKS on Outposts
 
+  outpost_arn   = element(tolist(data.aws_outposts_outposts.this.arns), 0)
+  instance_type = element(tolist(data.aws_outposts_outpost_instance_types.this.instance_types), 0)
+
   tags = {
     Example    = local.name
     GithubRepo = "terraform-aws-eks"
@@ -32,84 +36,159 @@ locals {
 module "eks" {
   source = "../.."
 
-  cluster_name                   = local.name
-  cluster_version                = local.cluster_version
-  cluster_endpoint_public_access = true
+  cluster_name    = local.name
+  cluster_version = local.cluster_version
+
+  cluster_endpoint_public_access  = false # Not available on Outpost
+  cluster_endpoint_private_access = true
+
+  vpc_id     = data.aws_vpc.this.id
+  subnet_ids = data.aws_subnets.this.ids
 
   outpost_config = {
-    control_plane_instance_type = var.outpost_instance_type
-    outpost_arns                = [tolist(data.aws_outposts_outposts.this.arns)[0]]
+    control_plane_instance_type = local.instance_type
+    outpost_arns                = [local.outpost_arn]
   }
 
-  cluster_addons = {
-    coredns = {
-      most_recent = true
+  # Extend cluster security group rules
+  cluster_security_group_additional_rules = {
+    ingress_vpc_https = {
+      description = "Remote host to control plane"
+      protocol    = "tcp"
+      from_port   = 443
+      to_port     = 443
+      type        = "ingress"
+      cidr_blocks = [data.aws_vpc.this.cidr_block]
     }
-    kube-proxy = {
-      most_recent = true
+    inress_nodes_ephemeral_ports_tcp = {
+      description                = "To node 1025-65535"
+      protocol                   = "tcp"
+      from_port                  = 1025
+      to_port                    = 65535
+      type                       = "ingress"
+      source_node_security_group = true
     }
-    vpc-cni = {
-      most_recent              = true
-      service_account_role_arn = module.vpc_cni_irsa.iam_role_arn
+    egress_nodes_ephemeral_ports_tcp = {
+      description                = "To node 1025-65535"
+      protocol                   = "tcp"
+      from_port                  = 1025
+      to_port                    = 65535
+      type                       = "egress"
+      source_node_security_group = true
     }
   }
 
-  # Encryption key
-  create_kms_key = true
-  cluster_encryption_config = {
-    resources = ["secrets"]
+  # Extend node-to-node security group rules
+  node_security_group_additional_rules = {
+    ingress_vpc_all = {
+      description = "VPC"
+      protocol    = "-1"
+      from_port   = 0
+      to_port     = 0
+      type        = "ingress"
+      cidr_blocks = [data.aws_vpc.this.cidr_block]
+    }
+    ingress_self_all = {
+      description = "Node to node all ports/protocols"
+      protocol    = "-1"
+      from_port   = 0
+      to_port     = 0
+      type        = "ingress"
+      self        = true
+    }
+    egress_all = {
+      description      = "Node all egress"
+      protocol         = "-1"
+      from_port        = 0
+      to_port          = 0
+      type             = "egress"
+      cidr_blocks      = ["0.0.0.0/0"]
+      ipv6_cidr_blocks = ["::/0"]
+    }
   }
-  kms_key_deletion_window_in_days = 7
-  enable_kms_key_rotation         = true
-
-  create_cluster_security_group = false
-  create_node_security_group    = false
-  subnet_ids                    = [tolist(data.aws_subnets.this.ids)[0]]
-
-  manage_aws_auth_configmap = true
-
-  eks_managed_node_group_defaults = {
-    instance_types = [var.outpost_instance_type]
 
+  self_managed_node_group_defaults = {
     attach_cluster_primary_security_group = true
+
+    iam_role_additional_policies = {
+      AmazonSSMManagedInstanceCore       = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+      AmazonEC2ContainerRegistryReadOnly = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
+    }
   }
 
-  eks_managed_node_groups = {
+  self_managed_node_groups = {
     outpost = {
       name = local.name
+
+      min_size      = 2
+      max_size      = 5
+      desired_size  = 3
+      instance_type = local.instance_type
     }
   }
 
+  # We need to add the node group IAM role to the aws-auth configmap
+  create_aws_auth_configmap = true
+
   tags = local.tags
 }
 
+resource "kubernetes_storage_class_v1" "this" {
+  metadata {
+    name = "ebs-sc"
+    annotations = {
+      "storageclass.kubernetes.io/is-default-class" = "true"
+    }
+  }
+
+  storage_provisioner    = "ebs.csi.aws.com"
+  volume_binding_mode    = "WaitForFirstConsumer"
+  allow_volume_expansion = true
+
+  parameters = {
+    type      = "gp2"
+    encrypted = "true"
+  }
+}
+
 ################################################################################
 # Supporting Resources
 ################################################################################
 
 data "aws_outposts_outposts" "this" {}
 
-data "aws_subnets" "this" {
+data "aws_outposts_outpost_instance_types" "this" {
+  arn = local.outpost_arn
+}
+
+# This just grabs the first Outpost and returns its subnets
+data "aws_subnets" "lookup" {
   filter {
     name   = "outpost-arn"
-    values = [tolist(data.aws_outposts_outposts.this.arns)[0]]
+    values = [local.outpost_arn]
   }
 }
 
-module "vpc_cni_irsa" {
-  source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
-  version = "~> 5.0"
+# This grabs a single subnet to reverse lookup those that belong to same VPC
+# This is whats used for the cluster
+data "aws_subnet" "this" {
+  id = element(tolist(data.aws_subnets.lookup.ids), 0)
+}
 
-  role_name_prefix      = "VPC-CNI-IRSA"
-  attach_vpc_cni_policy = true
-  vpc_cni_enable_ipv4   = true
+# These are subnets for the Outpost and restricted to the same VPC
+# This is whats used for the cluster
+data "aws_subnets" "this" {
+  filter {
+    name   = "outpost-arn"
+    values = [local.outpost_arn]
+  }
 
-  oidc_providers = {
-    main = {
-      provider_arn               = module.eks.oidc_provider_arn
-      namespace_service_accounts = ["kube-system:aws-node"]
-    }
+  filter {
+    name   = "vpc-id"
+    values = [data.aws_subnet.this.vpc_id]
   }
+}
 
-  tags = local.tags
+data "aws_vpc" "this" {
+  id = data.aws_subnet.this.vpc_id
 }