DevOps01: Setting Up a Local Kubernetes Cluster Using Terraform and Kind

When I want to run tests in a Kubernetes cluster, preparing the environment can be very challenging. Fortunately, we found that Kind is an easy-to-use tool for deploying Kubernetes clusters locally. This article will show you how to use Terraform and Kind to deploy a Kubernetes cluster on your local machine.

This is my github repo files：https://github.com/zeyangli/myiac/tree/main/k8s

1. Preparation

Install Docker

Install Terraform version 1.1.8 .

Install Kubectl (https://dl.k8s.io/release/v1.24.0/bin/linux/amd64/kubectl )

2. Write Kind resource

2.1. Define terraform providers

# providers
terraform {
  required_providers {
    kind = {
      source  = "tehcyx/kind"
      version = "0.0.12"
    }
    null = {
      source  = "hashicorp/null"
      version = "3.1.1"
    }
  }
}

provider "kind" {}

2.2. Define some variables

# define kubeconfig file path.
variable "kind_cluster_config_path" {
  type    = string
  default = "~/.kube/config"
}

# screen output kubeconfig context
output "kubeconfig" {
  value = kind_cluster.default.kubeconfig
}

2.3. Define kind_cluster configuration

resource "kind_cluster" "default" {
  name            = "devopscluster"                             # clusterName
  node_image      = "ccr.ccs.tencentyun.com/devopsvip/kind:v1.24.0"  #kindest/node:v1.24.0 kind docker image
  kubeconfig_path = pathexpand(var.kind_cluster_config_path)    # kubeconfig
  wait_for_ready  = true  # wait cluster ready
  
  # kind
  kind_config {
    kind        = "Cluster"
    api_version = "kind.x-k8s.io/v1alpha4"
    
    # Control node
    node {
      role = "control-plane"
      kubeadm_config_patches = [
        <<-EOT
          kind: InitConfiguration
          imageRepository: registry.aliyuncs.com/google_containers
          networking:
            serviceSubnet: 10.0.0.0/16
            apiServerAddress: "0.0.0.0"
          nodeRegistration:
            kubeletExtraArgs:
              node-labels: "ingress-ready=true"
          ---
          kind: KubeletConfiguration
          cgroupDriver: systemd
          cgroupRoot: /kubelet
          failSwapOn: false
        EOT
      ]
   extra_port_mappings {
        container_port = 80
        host_port      = 80
      }
      extra_port_mappings {
        container_port = 443
        host_port      = 443
      }
      extra_port_mappings {
        container_port = 6443
        host_port      = 6443
      }
      extra_mounts {
        host_path = "./resolv.conf"
        container_path = "/etc/resolv.conf"
      }
    }
    # worker 
    node {
      role = "worker"
      # extra_mounts {
      #   host_path = "./resolv.conf"
      #   container_path = "/etc/resolv.conf"
      # }
    }
    # worker 
    node {
      role = "worker"
      # extra_mounts {
      #   host_path = "./resolv.conf"
      #   container_path = "/etc/resolv.conf"
      # }
    }
  }
}

  depends_on = [kind_cluster.default]
}

2.4. Define ingress deploy commands

# null_resource to run some shell commands
resource "null_resource" "wait_for_instatll_ingress" {
  triggers = {
    key = uuid()
  }

  provisioner "local-exec" {
    command = <<EOF
      sleep 5  
      #kind load  docker-image k8s.gcr.io/ingress-nginx/controller:v1.2.0 --name devopscluster
      #kind load  docker-image k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1  --name devopscluster
      kind load docker-image ccr.ccs.tencentyun.com/devopsvip/kube-webhook-certgen:v1.1.1 --name devopscluster
      kind load docker-image ccr.ccs.tencentyun.com/devopsvip/controller:v1.2.0 --name devopscluster
      
      kubectl create ns ingress-nginx
      kubectl apply -f ingress.yaml -n ingress-nginx
      printf "\nWaiting for the nginx ingress controller...\n"
      kubectl wait --namespace ingress-nginx \
        --for=condition=ready pod \
       --selector=app.kubernetes.io/component=controller \
        --timeout=90s
    EOF
  }

3. Pull docker images

docker pull ccr.ccs.tencentyun.com/devopsvip/kind:v1.24.0
docker pull ccr.ccs.tencentyun.com/devopsvip/kube-webhook-certgen:v1.1.1
docker pull ccr.ccs.tencentyun.com/devopsvip/controller:v1.2.0

4. Deploy

4.1. Init terraform environment

terraform init

4.2. Apply terraform resource

terraform apply --auto-approve

if deploy succsed, you can see the screen output.

5. Check Kubernetes cluster

kubectl get node
NAME                         STATUS   ROLES           AGE     VERSION
devopscluster-control-plane   Ready    control-plane   2m53s   v1.24.0
devopscluster-worker          Ready    <none>          2m19s   v1.24.0
test-cluster-worker2         Ready    <none>          2m20s   v1.24.0

[root@zeyang-nuc-service k8s]# kubectl get pod -n ingress-nginx
NAME                                        READY   STATUS      RESTARTS   AGE
ingress-nginx-admission-create-fbh5n        0/1     Completed   0          5m41s
ingress-nginx-admission-patch-xttkh         0/1     Completed   0          5m41s
ingress-nginx-controller-7bd87766fb-lsv89   1/1     Running     0          5m41s