terraform-aws-eks icon indicating copy to clipboard operation
terraform-aws-eks copied to clipboard

Unable to add a new node group to the existing cluster using the eks module

Open vatsal-kavida opened this issue 1 year ago β€’ 0 comments

Description

I tried adding one more node group to my existing cluster, but as soon as I try to add it , as the vpc and subnets and nat gateways are not defined now so it starts deleting them also it tries to create a new cloudwatch group and kms group which results in error. Can anyone please help with the same . Attaching my manifests for better idea ->
main.tf

provider "aws" {
  region = local.region
}

provider "kubernetes" {
  host                   = module.eks.cluster_endpoint
  cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)

  exec {
    api_version = "client.authentication.k8s.io/v1beta1"
    command     = "aws"
    args        = ["eks", "get-token", "--cluster-name", module.eks.cluster_name]
  }
}

#data "aws_availability_zones" "available" {}
#data "aws_caller_identity" "current" {}

locals {
  name   = "mongodb-cluster"
  region = "ap-south-1"

  vpc_cidr = "10.0.0.0/16"
#  azs      = slice(data.aws_availability_zones.available.names, 0, 3)
#
#  private_subnets_cidrs = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
#  public_subnets_cidrs  = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"]

  tags = {
    cluster     = local.name
    environment = "test"
  }
}
#
module "vpc" {
  source  = "terraform-aws-modules/vpc/aws"
  version = "~> 4.0"

  name = local.name
  cidr = local.vpc_cidr
#
#  azs             = local.azs
#  private_subnets = local.private_subnets_cidrs
#  public_subnets  = local.public_subnets_cidrs

  enable_nat_gateway = true
  single_nat_gateway = true
  enable_dns_hostnames = true
  create_igw = true

  public_subnet_tags = {
    "kubernetes.io/cluster/${local.name}" = "shared"
    "kubernetes.io/role/elb"             = 1
  }

  private_subnet_tags = {
    "kubernetes.io/cluster/${local.name}" = "shared"
    "kubernetes.io/role/internal-elb"    = 1
  }

  tags = local.tags


  map_public_ip_on_launch = true
}

module "eks" {
  source = "terraform-aws-modules/eks/aws"

  cluster_name                            = local.name
  vpc_id                                  = "vpc-0e6bb503h9b47432a"
  subnet_ids                              = [
    "subnet-002f83c0l15571d77", "subnet-0c4c958055996713a", "subnet-06900kdad702c4350", "subnet-0pfd8fa63f0d4b1d1",
    "subnet-0bw6ff0577a64885d", "subnet-08f7ezb78b8d0d8bd"
  ]
  cluster_security_group_additional_rules = {
    egress_nodes_ephemeral_ports_tcp = {
      description                = "To node 1025-65535"
      protocol                   = "tcp"
      from_port                  = 1025
      to_port                    = 65535
      type                       = "egress"
      source_node_security_group = true
    }
  }
  node_security_group_additional_rules = {
    ingress_self_all = {
      description = "Node to node all ports/protocols"
      protocol    = "-1"
      from_port   = 0
      to_port     = 0
      type        = "ingress"
      self        = true
    }
    egress_all = {
      description = "Node all egress"
      protocol    = "-1"
      from_port   = 0
      to_port     = 0
      type        = "egress"
      cidr_blocks = ["0.0.0.0/0"]
    }
  }
  #  cluster_addons = {
  #    coredns = {
  #      most_recent = true
  #    }
  #    kube-proxy = {
  #      most_recent = true
  #    }
  #    vpc-cni = {
  #      most_recent = true
  #    }
  #  }

  eks_managed_node_group_defaults = {
    ami_type                              = "AL2_x86_64"
    instance_types                        = ["m5.large"]
    attach_cluster_primary_security_group = false
  }

  eks_managed_node_groups = {
    copilot = {
      min_size       = 1
      max_size       = 3
      desired_size   = 1
      capacity_type  = "SPOT"
      instance_types = ["r6a.xlarge"]
      labels         = {
        Environment = "test"
        Deployment  = "copilot"
        lifecycle   = "Ec2"
        spot        = "true"
        node-class  = "worker-node"
        role        = "managed-nodes"
      }

      cluster_autoscaler_settings = {
        scale_down_utilization_threshold = 0.9
        scan_interval                    = "10s"
        policies                         = [
          {
            name  = "custom-policy"
            rules = [
              {
                action = "add"
                type   = "PodsViolatingConstraints"
              },
            ]
          },
        ]
      }
      taints = {
        dedicated = {
          key    = "normalInstance"
          value  = "true"
          effect = "PREFER_NO_SCHEDULE"
        }
      }
      tags = {
        Deployment = "copilot"
      }
    }
  }
}
#resource "aws_security_group" "public" {
#  name_prefix = "public"
#  vpc_id = module.vpc.vpc_id
#
#  ingress {
#    from_port   = 0
#    to_port     = 0
#    protocol    = "-1"
#    self        = true
#  }
#
#  egress {
#    from_port   = 0
#    to_port     = 0
#    protocol    = "-1"
#    self        = true
#    cidr_blocks = ["0.0.0.0/0"]
#  }
#
#  tags = local.tags
#}

output.tf

################################################################################
# Cluster
################################################################################

output "cluster_arn" {
  description = "The Amazon Resource Name (ARN) of the cluster"
  value       = module.eks.cluster_arn
}

output "cluster_certificate_authority_data" {
  description = "Base64 encoded certificate data required to communicate with the cluster"
  value       = module.eks.cluster_certificate_authority_data
}

output "cluster_endpoint" {
  description = "Endpoint for your Kubernetes API server"
  value       = module.eks.cluster_endpoint
}

output "cluster_id" {
  description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts"
  value       = module.eks.cluster_id
}

output "cluster_name" {
  description = "The name of the EKS cluster"
  value       = module.eks.cluster_name
}

output "cluster_oidc_issuer_url" {
  description = "The URL on the EKS cluster for the OpenID Connect identity provider"
  value       = module.eks.cluster_oidc_issuer_url
}

output "cluster_platform_version" {
  description = "Platform version for the cluster"
  value       = module.eks.cluster_platform_version
}

output "cluster_status" {
  description = "Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED`"
  value       = module.eks.cluster_status
}

output "cluster_security_group_id" {
  description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console"
  value       = module.eks.cluster_security_group_id
}


################################################################################
# Security Group
################################################################################

output "cluster_security_group_arn" {
  description = "Amazon Resource Name (ARN) of the cluster security group"
  value       = module.eks.cluster_security_group_arn
}

################################################################################
# IRSA
################################################################################

output "oidc_provider" {
  description = "The OpenID Connect identity provider (issuer URL without leading `https://`)"
  value       = module.eks.oidc_provider
}

output "oidc_provider_arn" {
  description = "The ARN of the OIDC Provider if `enable_irsa = true`"
  value       = module.eks.oidc_provider_arn
}

output "cluster_tls_certificate_sha1_fingerprint" {
  description = "The SHA1 fingerprint of the public key of the cluster's certificate"
  value       = module.eks.cluster_tls_certificate_sha1_fingerprint
}

################################################################################
# IAM Role
################################################################################

output "cluster_iam_role_name" {
  description = "IAM role name of the EKS cluster"
  value       = module.eks.cluster_iam_role_name
}

output "cluster_iam_role_arn" {
  description = "IAM role ARN of the EKS cluster"
  value       = module.eks.cluster_iam_role_arn
}

output "cluster_iam_role_unique_id" {
  description = "Stable and unique string identifying the IAM role"
  value       = module.eks.cluster_iam_role_unique_id
}

################################################################################
# EKS Addons
################################################################################

#output "cluster_addons" {
#  description = "Map of attribute maps for all EKS cluster addons enabled"
#  value       = module.eks.cluster_addons
#}

################################################################################
# EKS Identity Provider
################################################################################

output "cluster_identity_providers" {
  description = "Map of attribute maps for all EKS identity providers enabled"
  value       = module.eks.cluster_identity_providers
}


################################################################################
# EKS Managed Node Group
################################################################################

output "eks_managed_node_groups" {
  description = "Map of attribute maps for all EKS managed node groups created"
  value       = module.eks.eks_managed_node_groups
}

output "eks_managed_node_groups_autoscaling_group_names" {
  description = "List of the autoscaling group names created by EKS managed node groups"
  value       = module.eks.eks_managed_node_groups_autoscaling_group_names
}


################################################################################
# Additional
################################################################################
```
version.tf 

terraform { required_providers { aws = { source = "hashicorp/aws" version = ">= 4.57" } kubernetes = { source = "hashicorp/kubernetes" version = ">= 2.10" } random = { source = "hashicorp/random" version = "~> 3.4.3" }

tls = {
  source  = "hashicorp/tls"
  version = "~> 4.0.4"
}

cloudinit = {
  source  = "hashicorp/cloudinit"
  version = "~> 2.2.0"
}

} required_version = "~> 1.3" }

vatsal-kavida avatar Feb 07 '24 20:02 vatsal-kavida