☸️ نحوه استقرار یک خوشه مقرون به صرفه AWS/EKS Kubernetes با استفاده از Terraform در سال 2023

variable "region" {
  description = "Cluster region"
  default = "eu-west-x"
}

variable "cluster_name" {
  description = "Name of the EKS cluster"
  default     = "my-project"
}

variable "kubernetes_version" {
  description = "Cluster Kubernetes version"
  default     = "1.24"
}

# Being in this list is required to see Kubernetes resources in AWS console
variable "aws_auth_users" {
  description = "Developers with access to the dev K8S cluster and the container registries"
  default = [
    {
      userarn  = "arn:aws:iam::xxx:user/user.name1"
      username = "user.name1"
      groups   = ["system:masters"]
    },
    {
      userarn  = "arn:aws:iam::xxx:user/user.name2"
      username = "user.name2"
      groups   = ["system:masters"]
    }
  ]
}

provider "aws" {
  region = var.region
  # prerequisite locally : aws configure --profile <cluster-name>
  profile = var.cluster_name
}

provider "kubernetes" {
  host                   = module.eks.cluster_endpoint
  cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)

  exec {
    api_version = "client.authentication.k8s.io/v1beta1"
    command     = "aws"
    # This requires the awscli to be installed locally where Terraform is executed
    args = ["--profile", var.cluster_name, "eks", "get-token", "--cluster-name", var.cluster_name]
  }
}

locals {
  tags = {
    Environment   = "NON-PROD"
    creation-date = "01/02/2023" # a variable would update the value on each tf apply
  }
}

data "aws_caller_identity" "current" {}

module "vpc" {

  source  = "terraform-aws-modules/vpc/aws"
  version = "~> 3.0"
  name    = var.cluster_name
  cidr    = "10.0.0.0/16" # Last IP : 10.0.255.255
  azs     = ["${var.region}a", "${var.region}b", "${var.region}c"]
  # use https://www.ipaddressguide.com/cidr
  # /19 : 8,192 IPs
  private_subnets      = ["10.0.0.0/19", "10.0.32.0/19", "10.0.64.0/19"]    # No hole in IP ranges
  public_subnets       = ["10.0.96.0/19", "10.0.128.0/19", "10.0.160.0/19"] # No hole in IP ranges
  enable_nat_gateway   = true
  single_nat_gateway   = true
  enable_dns_hostnames = true

  public_subnet_tags = {
    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
    "kubernetes.io/role/elb"                    = "1"
  }

  private_subnet_tags = {
    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
    "kubernetes.io/role/internal-elb"           = "1"
  }

  tags = local.tags

}

module "eks" {

  source = "terraform-aws-modules/eks/aws"

  cluster_name                    = var.cluster_name
  cluster_version                 = var.kubernetes_version
  cluster_endpoint_private_access = true
  cluster_endpoint_public_access  = true

  cluster_addons = {
    coredns = {
      most_recent = true

      timeouts = {
        create = "2m" # default 20m. Times out on first launch while being effectively created
      }
    }
    kube-proxy = {
      most_recent = true
    }
    vpc-cni = {
      most_recent = true
    }
    aws-ebs-csi-driver = {
      most_recent = true
    }
  }

  vpc_id     = module.vpc.vpc_id
  subnet_ids = module.vpc.private_subnets

  # Self managed node groups will not automatically create the aws-auth configmap so we need to
  create_aws_auth_configmap = true
  manage_aws_auth_configmap = true

  aws_auth_users = var.aws_auth_users

  enable_irsa = true

  node_security_group_additional_rules = {
    ingress_self_all = {
      description = "Node to node all ports/protocols"
      protocol    = "-1"
      from_port   = 0
      to_port     = 0
      type        = "ingress"
      self        = true
    }
    egress_all = { # by default, only https urls can be reached from inside the cluster
      description      = "Node all egress"
      protocol         = "-1"
      from_port        = 0
      to_port          = 0
      type             = "egress"
      cidr_blocks      = ["0.0.0.0/0"]
      ipv6_cidr_blocks = ["::/0"]
    }
  }

  self_managed_node_group_defaults = {

    # enable discovery of autoscaling groups by cluster-autoscaler
    autoscaling_group_tags = {
      "k8s.io/cluster-autoscaler/enabled" : true,
      "k8s.io/cluster-autoscaler/${var.cluster_name}" : "owned",
    }

    # from https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2207#issuecomment-1220679414
    # to avoid "waiting for a volume to be created, either by external provisioner "ebs.csi.aws.com" or manually created by system administrator"
    iam_role_additional_policies = {
      AmazonEBSCSIDriverPolicy = "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy"
    }

  }

  # possible values : https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/node_groups.tf
  self_managed_node_groups = {

    default_node_group = {
      create = false
    }

    # fulltime-az-a = {
    #   name                 = "fulltime-az-a"
    #   subnets              = [module.vpc.private_subnets[0]]
    #   instance_type        = "t3.medium"
    #   desired_size         = 1
    #   bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=normal'"
    # }

    spot-az-a = {
      name       = "spot-az-a"
      subnet_ids = [module.vpc.private_subnets[0]] # only one subnet to simplify PV usage
      # availability_zones = ["${var.region}a"] # conflict with previous option. TODO try subnet_ids=null at creation (because at modification it fails)

      desired_size         = 2
      min_size             = 1
      max_size             = 10
      bootstrap_extra_args = "--kubelet-extra-args '--node-labels=node.kubernetes.io/lifecycle=spot'"

      use_mixed_instances_policy = true
      mixed_instances_policy = {
        instances_distribution = {
          on_demand_base_capacity                  = 0
          on_demand_percentage_above_base_capacity = 0
          spot_allocation_strategy                 = "lowest-price" # "capacity-optimized" described here : https://aws.amazon.com/blogs/compute/introducing-the-capacity-optimized-allocation-strategy-for-amazon-ec2-spot-instances/
        }

        override = [
          {
            instance_type     = "t3.xlarge"
            weighted_capacity = "1"
          },
          {
            instance_type     = "t3a.xlarge"
            weighted_capacity = "1"
          },
        ]
      }

    }

  }

  tags = local.tags
}

resource "aws_ecr_repository" "module-a" {
  name = "my-app/module-a"
}

resource "aws_ecr_repository" "module-b" {
  name = "my-app/module-b"
}

resource "aws_ecr_repository" "module-c" {
  name = "my-app/module-c"
}

resource "aws_s3_bucket" "bucket" {
  bucket = "${var.cluster_name}-bucket"

  tags = local.tags
}

resource "aws_s3_bucket_acl" "bucket_acl" {
  bucket = aws_s3_bucket.bucket.id
  acl    = "private"
}

data "aws_iam_policy_document" "role_policy" {
  statement {
    actions = ["sts:AssumeRoleWithWebIdentity"]
    effect  = "Allow"

    condition {
      test     = "StringLike"
      variable = "${replace(module.eks.cluster_oidc_issuer_url, "https://", "")}:sub"
      values   = ["system:serviceaccount:*:backend"] # system:serviceaccount:<K8S_NAMESPACE>:<K8S_SERVICE_ACCOUNT>
    }

    principals {
      identifiers = ["arn:aws:iam::${var.aws_account_id}:oidc-provider/${replace(module.eks.cluster_oidc_issuer_url, "https://", "")}"]
      type        = "Federated"
    }
  }
}

data "aws_iam_policy_document" "s3_policy" {
  statement {
    actions = [
      "s3:ListAllMyBuckets",
    ]

    resources = [
      "*",
    ]
  }
  statement {
    actions = [
      "s3:*",
    ]

    resources = [
      aws_s3_bucket.bucket.arn,
      "${aws_s3_bucket.bucket.arn}/*"
    ]
  }
}

resource "aws_iam_role" "role" {
  assume_role_policy = data.aws_iam_policy_document.role_policy.json
  name               = "${var.cluster_name}-backend-role"
}

resource "aws_iam_policy" "policy" {
  name   = "${var.cluster_name}-backend-policy"
  path   = "https://dev.to/"
  policy = data.aws_iam_policy_document.s3_policy.json
}

resource "aws_iam_role_policy_attachment" "attach" {
  policy_arn = aws_iam_policy.policy.arn
  role       = aws_iam_role.role.name
}

output "update_local_context_command" {
  description = "Command to update local kube context"
  value       = "aws --profile ${var.cluster_name} eks update-kubeconfig --name=${var.cluster_name} --alias=${var.cluster_name} --region=${var.region}"
}