Terraform plan for ~20 kubernetes_manifest resource takes 20 mins or times out

[varunthakur2480]

Terraform Version, Provider Version and Kubernetes Version

Terraform v0.14.10
+ provider registry.terraform.io/hashicorp/google v4.17.0
+ provider registry.terraform.io/hashicorp/google-beta v4.17.0
+ provider registry.terraform.io/hashicorp/kubernetes v2.20.0
+ provider registry.terraform.io/hashicorp/vault v3.1.1

Kubernetes version: 1.24

Affected Resource(s)

Terraform Configuration Files

## Modules code


##
## GKE IaC
##
resource "kubernetes_manifest" "appplication_source" {

  manifest = {
    apiVersion = "source.toolkit.fluxcd.io/v1beta2"
    kind       = "GitRepository"
    metadata = {
      name      = var.iac_repo_name
      namespace = var.namespace

      finalizers = ["finalizers.fluxcd.io"]
    }
    spec = {
      gitImplementation = "go-git"
      interval          = "1m0s"
      url               = var.iac_gitlab_repo
      ref = {
        branch = var.iac_git_branch
        tag    = var.iac_git_tag
      }
      secretRef = {
        name = var.iac_flux_secret_name
      }
      timeout = "20s"
    }
  }
  field_manager {
    force_conflicts = true
  }

}

resource "kubernetes_manifest" "application_kustomize" {
  manifest = {
    apiVersion = "kustomize.toolkit.fluxcd.io/v1beta2"
    kind       = "Kustomization"
    metadata = {
      name      = var.iac_repo_name
      namespace = var.namespace
    }
    spec = {
      force              = var.flux_force
      interval           = "1m0s"
      path               = var.git_path
      suspend            = var.flux_suspend
      prune              = true
      serviceAccountName = "flux"
      sourceRef = {
        kind      = "GitRepository"
        name      = var.iac_repo_name
        namespace = var.namespace
      }
      targetNamespace = var.namespace
      validation      = "server"
    }
  }
  field_manager {
    force_conflicts = true
  }
}

variable "env" {
}

variable "env_region" {
  description = "env region"
}

variable "flux_force" {
  type        = bool
  description = "Flux config settings for force option"
  default     = true
}

variable "flux_suspend" {
  type        = bool
  description = "Flux config settings for suspend option"
  default     = false
}

variable "git_path" {
  description = "git path for kustomize"
}

variable "iac_flux_secret_name" {
  description = "Iac flux secret name"
  default     = "iac-flux-secret"
}

variable "iac_git_branch" {
  default = "master"
}

# remove default to enforce tag usage
variable "iac_git_tag" {
  description = "The Git tag to checkout, takes precedence over git_branch"
#  default     = ""
}

variable "iac_gitlab_repo" {
  description = "URL of the git repo"
}

variable "iac_repo_name" {
  description = "name of the git repo"
  default     = "gke-iac"
}

variable "namespace" {
  description = "Kubernetes namespace name"
}
# versions.tf

terraform {
  required_version = ">= 0.14"
  required_providers {
    google = {
      source  = "hashicorp/google"
      version = "3.71.0"
    }
    google-beta = {
      source  = "hashicorp/google-beta"
      version = "3.71.0"
    }
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "2.7.1"
    }
    vault = {
      source  = "hashicorp/vault"
      version = "3.1.1"
    }
  }
}

# Invoking resource
# This example is for managing namespace flux config using terraform 

data "vault_generic_secret" "gke_uat0" {
  path = "iac-pipeline/kv/kubernetes/uat1/euw2/app/uat/uat1-e2-uat-uat0/namespace-iac"
}

data "google_container_cluster" "gke_uat0" {
  location = local.region
  name     = data.vault_generic_secret.gke_uat0.data["cluster_name"]
  project  = data.vault_generic_secret.gke_uat0.data["cluster_project_id"]
}

provider "kubernetes" {
  alias                  = "gke_uat0"
  host                   = "https://${data.google_container_cluster.gke_uat0.endpoint}"
  token                  = data.vault_generic_secret.gke_uat0.data["k8s_sa_token"]
  cluster_ca_certificate = base64decode(data.google_container_cluster.gke_uat0.master_auth.0.cluster_ca_certificate)
}

module "flux_uat0" {
  providers = {
    kubernetes = kubernetes.gke_uat0
  }
  source = "../../../modules/flux-setup"

  env                  = var.env
  env_region           = local.env_region
  flux_force           = var.flux_force
  flux_suspend         = var.flux_suspend
  git_path             = "kustomize/uat0/gke"
  iac_flux_secret_name = local.iac_flux_secret_name
  iac_git_tag          = var.uat0_iac_git_tag
  iac_gitlab_repo      = local.iac_gitlab_repo
  iac_repo_name        = local.iac_repo_name
  namespace            = "uat1-e2-uat-app-uat0"
}

# environment specific uncommon variables, don't change default here
variable "uat0_iac_git_tag" {
  default = ""
}

#Main.tf
locals {
  env_region           = "e2"
  iac_flux_secret_name = "iac-flux-secret"
  iac_gitlab_repo      = data.terraform_remote_state.project.outputs.gke_iac_repo_clone_url
  iac_repo_name        = "gke-iac"
  region               = "europe-west2"
}

terraform {
  backend "remote" {
    hostname     = "terraform-enterprise.****"
    organization = "nwm-non-prod-v2"

    workspaces {
      name = "uat1-app-gkens"
    }
  }
}

# Secrets and credentials come from Vault
provider "vault" {

  auth_login {
    path = "auth/approle/login"

    parameters = {
      role_id   = var.vault_approle
      secret_id = var.vault_approle_secret
    }
  }
}

data "vault_generic_secret" "gke_cluster_viewer" {
  path = "test/gcp/token/uat1-euw2-terraform-gke-cluster-viewer"
}


provider "google" {
  access_token = data.vault_generic_secret.gke_cluster_viewer.data["token"]
  region       = "europe-west2"
}

data "google_client_config" "default" {}



data "terraform_remote_state" "project" {
  backend = "remote"

  config = {
    hostname     = "terraform-enterprise.platform.nwminfra.net"
    organization = "nwm-non-prod-v2"
    workspaces = {
      name = "uat-app-project"
    }
  }
}
#Varinables .tf
variable "cmdb_id" {
  description = "The CRISP `cmdb_id` for your application"
}

variable "application" {
  description = "The name of your application within CRISP"
}

variable "cost_center" {
  description = "The cost center for your application / project / team"
}

variable "env" {
  description = "The value of the environment for this"
  default     = "uat1"
}

variable "flux_suspend" {
  type        = bool
  description = "Flux config settings for suspend option"
  default     = false
}

variable "flux_force" {
  type        = bool
  description = "Flux config settings for force option"
  default     = true
}

variable "iac_git_branch" {
  type        = string
  description = "Flux config branch setting to be used for resource apply by flux"

  default = "master"
}

variable "iac_git_tag" {
  type = string

  description = "The Git tag to checkout, takes precedence over git_branch"
  default     = ""
}

variable "owner" {
  description = "The CRISP `owner` of your application"
}


variable "vault_approle" {
  description = "The approle to be used when authenticating with Vault"
}

variable "vault_approle_secret" {
  description = "The approle secret to be used when authenticating with Vault"
}

``

### Debug Output
<!--Please provider a link to a GitHub Gist containing the complete debug output: https://www.terraform.io/docs/internals/debugging.html. Please do NOT paste the debug output in the issue; just paste a link to the Gist.-->

### Panic Output
<!-- Debug log to be uploaded-->

### Steps to Reproduce
<!-- Please list the steps required to reproduce the issue, for example:
1. `terraform init
2. terraform plan` -->

### Expected Behavior
What should have happened?

### Actual Behavior
What actually happened?

### Important Factoids
<!-- This happens only for flux custom resources being managed using kubernetes_manifest-->



### Community Note
<!--- Please keep this note for the community --->
* Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request
* If you are interested in working on this issue or have submitted a pull request, please leave a comment

[varunthakur2480]

run-34cdAV2VcLFen9NY-plan-log.txt

[github-actions[bot]]

Marking this issue as stale due to inactivity. If this issue receives no comments in the next 30 days it will automatically be closed. If this issue was automatically closed and you feel this issue should be reopened, we encourage creating a new issue linking back to this one for added context. This helps our maintainers find and focus on the active issues. Maintainers may also remove the stale label at their discretion. Thank you!