ã¯ããã« ããã«ã¡ã¯ãã€ãããŒã·ã§ã³ã»ã³ã¿ãŒã®éŽã¶å¶ºã§ãã engineers.ntt.com engineers.ntt.com 第1åã第2åã«åŒãç¶ãAWS Outposts ã©ãã¯ã«ã€ããŠç޹ä»ããŠãããŸãã æ¬èšäºã§ã¯ã Terraform ããçšããŠOutpostsäžã§ãªã³ãã¬ç°å¢ããã®ã¿ç®¡çã»ã¢ã¯ã»ã¹å¯èœãªPrivate Elastic Kubernetes Service(EKS) ãæ§ç¯ããæ¹æ³ã玹ä»ããŸãã Terraform Terraformãšã¯ã HashCorp ãæäŸããã€ã³ãã©ãã³ãŒãåããŠèªåæ§ç¯ãå¯èœãšããInfrastructure as Code(IaC)ãå®çŸããããã®ããŒã«ã§ããOutpostsãTerraformã§IaCåããããã«ã¯ã察å¿ããåãªãœãŒã¹ã« outpost_arn ãæž¡ãããšã§Outpostsäžã®ãªãœãŒã¹ãäœæãããŸãã 以äžã¯subnetã®äžäŸã§ãã 1 resource "aws_subnet" "main" { vpc_id = aws_vpc.main.id cidr_block = "10.0.3.0/24" outpost_arn = "arn:aws:outposts:ap-northeast-1:1234567890:outpost/op-1234567890" } outpost_arn - (Optional) The Amazon Resource Name (ARN) of the Outpost. CDKãšæ¯èŒããŠTerraformã¯AWS APIãå©çšããŠããã®ã§Outpostsã®ãªãœãŒã¹ã¯æ¯èŒç察å¿ãããŠãããšæãããŸããä»åŸæ°ãããªãœãŒã¹ãOutpostsã§å©çšå¯èœã«ãªã£ãå Žåã察å¿ã¹ããŒãã¯æ©ããšäºæ³ãããŸãããã äžæ¹ã§ãCDKã®ãããªé«ã¬ãã«ãªãªãœãŒã¹äœæãããããšãé£ããã®ã§ãªãœãŒã¹ãAPIã®ä»æ§ãææ¡ããå¿
èŠããããŸãããŸããªãœãŒã¹ã®èªèšŒæ
å ±ã»ç¶æ
管çãªã©ãèããå¿
èŠããããŸãã ãªã³ãã¬ç°å¢äžã§ãããŒãžãk8sãµãŒãã¹ãæ§ç¯ããã¢ãŒããã¯ã㣠äžèšã®ãããªãªã³ãã¬ç°å¢ã«ãããŠããããŒãžãk8sãµãŒãã¹ãå©çšããã¢ãŒããã¯ãã£ãæ§ç¯ããŸããOutpostsäžã«Auto Scale Groupãèšçœ®ããŠã»ã«ããããŒãžãåããŒããšããŠããããEC2ãEKS Clusterã«ç»é²ãããŸããOutpostsãšãªã³ãã¬ç°å¢ãšã®éä¿¡ã¯å
šãŠLocal Gatewayçµç±ã§æ¥ç¶ãããŸãã ãã®æ§æã®ç¹åŸŽãšããŠEKS Clusterã®APIãµãŒããPrivateã¢ã¯ã»ã¹ã®ã¿èš±å¯ããããšã§ãBastion(èžã¿å°)çµç±ã§ã®ã¿ã¯ã©ã¹ã¿ã®æäœãå¯èœãšããã»ãã¥ã¢ãªç¹ãæããããŸããããã«ãã¢ããªã±ãŒã·ã§ã³ããªã³ãã¬ç°å¢ã®ã¿ããã¢ã¯ã»ã¹å¯èœãªãã瀟å
ã«ã®ã¿å
¬éããç§å¿æ§ã®é«ãã·ã¹ãã ãæ§ç¯ã§ããŸãã å®è£
以äžã®ãããªæ§æã§ãããããã®tfãã¡ã€ã«ã«ãªãœãŒã¹ãå®çŸ©ããŸãã main.tf Providerãå倿°ãªã©ã®å
šäœãèšèŒ network.tf ãããã¯ãŒã¯ã«é¢ãããã®ãèšèŒ eks_cluster.tf EKS Clusterã«é¢ãããã®ãèšèŒ eks_node.tf EKS Nodeã«é¢ãããã®ãèšèŒ security_group.tf Security Groupã«ããéä¿¡ã®èš±å¯ã«é¢ãããã®ãèšèŒ bastion.tf èžã¿å°ãµãŒãã«é¢ãããã®ãèšèŒ output.tf k8sã¢ããªã±ãŒã·ã§ã³ã®ãããã€æã«äœ¿çšãããã®ãèšèŒ main.tf terraform { required_providers { aws = { source = "hashicorp/aws" version = "~> 3.0" } } } provider "aws" { region = var.region } # Name tagãã«äžéšäœ¿çšããå€ variable "name" { default = "outposts-eks-sample" } # EKS Node ã®ã€ã³ã¹ã¿ã³ã¹ variable "instance_type" { default = "m5.large" } # EKS Node ã®disk(GB) variable "disk_size" { default = 100 } variable "region" { default = "ap-northeast-1" } # EKS Node ã®æãŸããããŒãæ° variable "desired_node" { default = 3 } # EKS Node ã®æäœããŒãæ° variable "min_node" { default = 3 } # EKS Node ã®æé«ããŒãæ° variable "max_node" { default = 3 } # Local Gatewayçµç±ããEKSã«ã¢ã¯ã»ã¹å¯èœãªãªã³ãã¬ç°å¢ã®CIDR variable "local_cidr_blocks" { default = [ "192.168.0.0/16" ] } locals { # outpostã®ARN outpost_arn = "arn:aws:outposts:ap-northeast-1:1234567890:outpost/op-1234567890" # Local Gatewayã®ID lgw_id = "lgw-1234567890" # Local Gatewayã®ã«ãŒãããŒãã«ã®ID lgw_rtb_id = "lgw-rtb-1234567890" # 顧客ææã®IPã¢ãã¬ã¹ customer_owned_ipv4_pool = "ipv4pool-coip-1234567890" # GPUã䜿çšããŠãããã©ãã gpu = replace (var.instance_type, "g4dn" , "!" ) != var.instance_type } # Bastion, EKS Nodeã«ã¢ã¯ã»ã¹å¯èœãªéµ resource "aws_key_pair" "key" { key_name = "$ { var.name } -key" public_key = file ( "~/.ssh/id_rsa.pub" ) } main.tfã«ã¯ AWS Provider ãšå倿°ãBastionã®éµãèšå®ããŸãã Outpostsã®ARN outpost_arn ã Local Gatewayã®ID lgw_id ãªã©ã¯ããã«èšå®ããŠãããŸãã Outpostsäžã§ãAuto Scaling groupã¯ä»æ§å¯èœã§ãããã®èšå®ã«çšããŠã max_node , min_node , desired_node ã®é¢ä¿ã¯æ¬¡ã®ç»åã®ããã«ãªã£ãŠãããŸãã 2 ä»åã¯3ããŒãã®æ§æã§äœæããŸãã network.tf resource "aws_vpc" "main" { cidr_block = "10.0.0.0/16" # VPCãšã³ããã€ã³ããæå¹ã«ããããã«dns hostnames, supportãèšå® enable_dns_hostnames = true enable_dns_support = true tags = { Name = "vpc-$ { var.name } " } } # EKS Clusterã®subnet resource "aws_subnet" "cluster01" { vpc_id = aws_vpc.main.id cidr_block = "10.0.1.0/24" tags = { Name = "subnet-cluster01-$ { var.name } " } } # EKS Clusterã®subnet resource "aws_subnet" "cluster02" { vpc_id = aws_vpc.main.id cidr_block = "10.0.2.0/24" tags = { Name = "subnet-cluster02-$ { var.name } " } } # Outpostsã®subnet resource "aws_subnet" "outposts_subnet01" { vpc_id = aws_vpc.main.id cidr_block = "10.0.3.0/24" outpost_arn = local.outpost_arn availability_zone = "ap-northeast-1a" tags = { Name = "subnet-outposts-subnet01-$ { var.name } " "kubernetes.io/cluster/$ { aws_eks_cluster.main.name } " = "owned" "kubernetes.io/role/elb" = "1" } } # ããã©ã«ãã«ãŒããLocal Gatewayã«èšå® resource "aws_route_table" "local_gateway" { vpc_id = aws_vpc.main.id route { cidr_block = "0.0.0.0/0" local_gateway_id = local.lgw_id } tags = { Name = "local-gateway-route-table-$ { var.name } " } } resource "aws_route_table_association" "outposts_subnet01" { subnet_id = aws_subnet.outposts_subnet01.id route_table_id = aws_route_table.local_gateway.id } data "aws_ec2_local_gateway_route_table" "main" { outpost_arn = local.outpost_arn } resource "aws_ec2_local_gateway_route_table_vpc_association" "main" { local_gateway_route_table_id = data.aws_ec2_local_gateway_route_table.main.id vpc_id = aws_vpc.main.id } # EKS Nodeã«èšå®ããEIPã®Pool resource "aws_eip" "main" { count = var.max_node customer_owned_ipv4_pool = local.customer_owned_ipv4_pool tags = { Name = "eip-node-$ { count.index } " } } # Bastion(èžã¿å°)ã®EIP resource "aws_eip" "bastion" { customer_owned_ipv4_pool = local.customer_owned_ipv4_pool tags = { Name = "eip-bastion-$ { var.name } " } } # åæèµ·åæã«EKS Nodeèªèº«ãEIPãèªãèšå®ããããã®EC2ãšã³ããã€ã³ããèšå® resource "aws_vpc_endpoint" "ec2" { vpc_id = aws_vpc.main.id service_name = "com.amazonaws.$ { var.region } .ec2" vpc_endpoint_type = "Interface" subnet_ids = [ aws_subnet.outposts_subnet01.id ] security_group_ids = [ aws_security_group.vpc_endpoint_ec2.id, ] private_dns_enabled = true } network.tfã«ã¯ãããã¯ãŒã¯ã«é¢ãããªãœãŒã¹ãå®çŸ©ããŸãã ãããã¯ãŒã¯æ§æãšããŠãPublic AWSäžã«subnetã2ã€ãOutpostsäžã«subnetã1ã€äœæããŸããOutpostsã®subnetã«EKSã®Worker Nodeãèšçœ®ãããŸããåºæ¬çã«Outpostsã®subnetã¯ããã©ã«ãã«ãŒããLocal Gatewayã«èšå®ããããšã§ã€ã³ã¿ãŒããããªã©ã®ã¢ã¯ã»ã¹ã¯ãªã³ãã¬ç°å¢çµç±ã§æ¥ç¶ãããããã«ããŸããããã«ãã瀟å
ã®ãããã¯ãŒã¯ããªã·ãŒã®é©çšãç£èŠãªã©ãå¯èœãšããŸãããŸããLocal Gatewayã«æ¥ç¶ããããã«ã¯é¡§å®¢ææã®IPã¢ãã¬ã¹customer owned IP address(CoIP)ãã¢ã¿ããããå¿
èŠãããããEKS NodeãBastionçšã®CoIPã®Elastic IP ã¢ãã¬ã¹(EIP)ãäœæããŠãããŸãã eks_cluster.tf # EKS Cluster resource "aws_eks_cluster" "main" { name = var.name role_arn = aws_iam_role.cluster_role.arn vpc_config { security_group_ids = [ aws_security_group.cluster.id ] subnet_ids = [ aws_subnet.cluster01.id, aws_subnet.cluster02.id ] # Private Accessã®ã¿ã«èšå®ããŠBastionçµç±ã§ã®ã¿æäœãã endpoint_public_access = false endpoint_private_access = true } depends_on = [ aws_iam_role_policy_attachment.eks_cluster_policy, aws_iam_role_policy_attachment.eks_service_policy, aws_iam_role_policy_attachment.eks_vpc_resource-controller, ] } resource "aws_iam_role" "cluster_role" { name = "eks-cluster-role-$ { var.name } " assume_role_policy = <<EOF { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Service": "eks.amazonaws.com" }, "Action": "sts:AssumeRole" } ] } EOF } resource "aws_iam_role_policy_attachment" "eks_cluster_policy" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy" role = aws_iam_role.cluster_role.name } resource "aws_iam_role_policy_attachment" "eks_service_policy" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKSServicePolicy" role = aws_iam_role.cluster_role.name } resource "aws_iam_role_policy_attachment" "eks_vpc_resource-controller" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKSVPCResourceController" role = aws_iam_role.cluster_role.name } # ALBã䜿çšããããã®IAM Policy resource "aws_iam_policy" "aws_load_balancer_controller" { name = "alb-policy-$ { var.name } " # from https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.2.1/docs/install/iam_policy.json policy = <<EOF { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "iam:CreateServiceLinkedRole", "ec2:DescribeAccountAttributes", "ec2:DescribeAddresses", "ec2:DescribeAvailabilityZones", "ec2:DescribeInternetGateways", "ec2:DescribeVpcs", "ec2:DescribeSubnets", "ec2:DescribeSecurityGroups", "ec2:DescribeInstances", "ec2:DescribeNetworkInterfaces", "ec2:DescribeTags", "ec2:GetCoipPoolUsage", "ec2:DescribeCoipPools", "elasticloadbalancing:DescribeLoadBalancers", "elasticloadbalancing:DescribeLoadBalancerAttributes", "elasticloadbalancing:DescribeListeners", "elasticloadbalancing:DescribeListenerCertificates", "elasticloadbalancing:DescribeSSLPolicies", "elasticloadbalancing:DescribeRules", "elasticloadbalancing:DescribeTargetGroups", "elasticloadbalancing:DescribeTargetGroupAttributes", "elasticloadbalancing:DescribeTargetHealth", "elasticloadbalancing:DescribeTags" ], "Resource": "*" }, { "Effect": "Allow", "Action": [ "cognito-idp:DescribeUserPoolClient", "acm:ListCertificates", "acm:DescribeCertificate", "iam:ListServerCertificates", "iam:GetServerCertificate", "waf-regional:GetWebACL", "waf-regional:GetWebACLForResource", "waf-regional:AssociateWebACL", "waf-regional:DisassociateWebACL", "wafv2:GetWebACL", "wafv2:GetWebACLForResource", "wafv2:AssociateWebACL", "wafv2:DisassociateWebACL", "shield:GetSubscriptionState", "shield:DescribeProtection", "shield:CreateProtection", "shield:DeleteProtection" ], "Resource": "*" }, { "Effect": "Allow", "Action": [ "ec2:AuthorizeSecurityGroupIngress", "ec2:RevokeSecurityGroupIngress" ], "Resource": "*" }, { "Effect": "Allow", "Action": [ "ec2:CreateSecurityGroup" ], "Resource": "*" }, { "Effect": "Allow", "Action": [ "ec2:CreateTags" ], "Resource": "arn:aws:ec2:*:*:security-group/*", "Condition": { "StringEquals": { "ec2:CreateAction": "CreateSecurityGroup" }, "Null": { "aws:RequestTag/elbv2.k8s.aws/cluster": "false" } } }, { "Effect": "Allow", "Action": [ "ec2:CreateTags", "ec2:DeleteTags" ], "Resource": "arn:aws:ec2:*:*:security-group/*", "Condition": { "Null": { "aws:RequestTag/elbv2.k8s.aws/cluster": "true", "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" } } }, { "Effect": "Allow", "Action": [ "ec2:AuthorizeSecurityGroupIngress", "ec2:RevokeSecurityGroupIngress", "ec2:DeleteSecurityGroup" ], "Resource": "*", "Condition": { "Null": { "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" } } }, { "Effect": "Allow", "Action": [ "elasticloadbalancing:CreateLoadBalancer", "elasticloadbalancing:CreateTargetGroup" ], "Resource": "*", "Condition": { "Null": { "aws:RequestTag/elbv2.k8s.aws/cluster": "false" } } }, { "Effect": "Allow", "Action": [ "elasticloadbalancing:CreateListener", "elasticloadbalancing:DeleteListener", "elasticloadbalancing:CreateRule", "elasticloadbalancing:DeleteRule" ], "Resource": "*" }, { "Effect": "Allow", "Action": [ "elasticloadbalancing:AddTags", "elasticloadbalancing:RemoveTags" ], "Resource": [ "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*", "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*" ], "Condition": { "Null": { "aws:RequestTag/elbv2.k8s.aws/cluster": "true", "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" } } }, { "Effect": "Allow", "Action": [ "elasticloadbalancing:AddTags", "elasticloadbalancing:RemoveTags" ], "Resource": [ "arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*", "arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*", "arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*", "arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*" ] }, { "Effect": "Allow", "Action": [ "elasticloadbalancing:ModifyLoadBalancerAttributes", "elasticloadbalancing:SetIpAddressType", "elasticloadbalancing:SetSecurityGroups", "elasticloadbalancing:SetSubnets", "elasticloadbalancing:DeleteLoadBalancer", "elasticloadbalancing:ModifyTargetGroup", "elasticloadbalancing:ModifyTargetGroupAttributes", "elasticloadbalancing:DeleteTargetGroup" ], "Resource": "*", "Condition": { "Null": { "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" } } }, { "Effect": "Allow", "Action": [ "elasticloadbalancing:RegisterTargets", "elasticloadbalancing:DeregisterTargets" ], "Resource": "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*" }, { "Effect": "Allow", "Action": [ "elasticloadbalancing:SetWebAcl", "elasticloadbalancing:ModifyListener", "elasticloadbalancing:AddListenerCertificates", "elasticloadbalancing:RemoveListenerCertificates", "elasticloadbalancing:ModifyRule" ], "Resource": "*" } ] } EOF } eks_cluster.tfã§ã¯ãEKS Clusterã«é¢ãããªãœãŒã¹ãå®çŸ©ããŠããŸãã ããã§ã¯ãEKS Clusterã®Private Accessãèšå®ããããšã§å
éšããã®ã¿APIãµãŒãã«ã¢ã¯ã»ã¹å¯èœãšãªããŸãããã®èšå®ãããããšã§ããã»ãã¥ã¢ãªk8sç°å¢ãå®çŸã§ããŸããéçšã®éã«ã¯Bastionçµç±ã§kubectl, helmãªã©ã®æäœãããŸãã eks_node.tf locals { # åæèµ·åæã«EKS Nodeèªèº«ãEIPãèªãèšå®ããŠãªã³ãã¬ç°å¢ã«å¯ŸããŠéä¿¡ãã node_user_data = <<EOF #!/bin/bash set -o xtrace export AWS_DEFAULT_REGION=$ { var.region } instance_id=$(curl 169 . 254 . 169 . 254 /latest/meta-data/instance-id/) # EIP Check function function check_eip () { RES=`aws ec2 describe-addresses --filters Name=allocation-id,Values=$ 1 | jq .Addresses [ 0 ] .InstanceId` if [ "$RES" = "null" ] ; then return 0 fi return 1 } eips= "$ { join ( " " , aws_eip.main.*.id) } " # EIP Attach for eip in $eips; do check_eip $eip if [ $? = 0 ] ; then aws ec2 associate-address --no-allow-reassociation --instance-id $instance_id --allocation-id $eip if [ $? = 0 ] ; then break fi fi done # EKS Node Bootstrap /etc/eks/bootstrap.sh $ { aws_eks_cluster.main.name } EOF node_role_name = "node_role_$ { var.name } " } data "aws_ssm_parameter" "eks_cpu_ami" { name = "/aws/service/eks/optimized-ami/1.21/amazon-linux-2/recommended/image_id" } data "aws_ssm_parameter" "eks_gpu_ami" { name = "/aws/service/eks/optimized-ami/1.21/amazon-linux-2-gpu/recommended/image_id" } resource "aws_launch_template" "node_template" { name_prefix = "eks_sample_node_group_template" image_id = local.gpu == true ? data.aws_ssm_parameter.eks_gpu_ami.value : data.aws_ssm_parameter.eks_cpu_ami.value instance_type = var.instance_type vpc_security_group_ids = [ aws_security_group.node.id ] key_name = aws_key_pair.key.id block_device_mappings { device_name = "/dev/xvda" ebs { volume_size = var.disk_size } } iam_instance_profile { name = aws_iam_instance_profile.instance_role.name } tag_specifications { resource_type = "instance" tags = { Name = "eks-sample-node" } } user_data = base64encode (local.node_user_data) } # EKS Nodeã®AutoScaleGroup resource "aws_autoscaling_group" "node_group" { vpc_zone_identifier = [ aws_subnet.outposts_subnet01.id ] desired_capacity = var.desired_node max_size = var.max_node min_size = var.min_node health_check_type = "EC2" wait_for_capacity_timeout = "120m" launch_template { id = aws_launch_template.node_template.id version = "$Latest" } tag { key = "kubernetes.io/cluster/$ { aws_eks_cluster.main.name } " value = "owned" propagate_at_launch = true } } resource "aws_iam_role" "node_group_role" { name = "node-group-role-$ { var.name } " assume_role_policy = <<EOF { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Service": "ec2.amazonaws.com" }, "Action": "sts:AssumeRole" } ] } EOF } resource "aws_iam_role_policy_attachment" "eks-worker-node-policy" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy" role = aws_iam_role.node_group_role.name } resource "aws_iam_role_policy_attachment" "eks-cni-policy" { policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" role = aws_iam_role.node_group_role.name } resource "aws_iam_role_policy_attachment" "eks-container-registry-read-only" { policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" role = aws_iam_role.node_group_role.name } resource "aws_iam_instance_profile" "instance_role" { name = local.node_role_name role = aws_iam_role.node_group_role.name } # EKS Nodeã«EIPãAttachããããã®IAM Policy resource "aws_iam_policy" "eip_attach_policy" { name = "eip-attach-policy-$ { var.name } " policy = <<EOF { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "ec2:DescribeAddresses", "ec2:AssociateAddress" ], "Resource": "*" } ] } EOF } resource "aws_iam_role_policy_attachment" "eip_attach_policy" { role = aws_iam_role.node_group_role.name policy_arn = aws_iam_policy.eip_attach_policy.arn } # ã»ã«ããããŒãžãåããŒããEKSã«ç»é²ãããããã®èšå®ãBastionã§ã®APIãµãŒãã«å¯Ÿããkubectlæäœã®èš±å¯èšå® resource "local_file" "aws_auth" { content = <<EOF apiVersion: v1 kind: ConfigMap metadata: name: aws-auth namespace: kube-system data: mapRoles: | - rolearn: $ { aws_iam_role.node_group_role.arn } username: system:node:{{EC2PrivateDNSName}} groups: - system:bootstrappers - system:nodes - rolearn: $ { aws_iam_role.bastion.arn } username: bastionrole groups: - system:masters EOF filename = "_aws-auth.yaml" } eks_node.tfã§ã¯ãEKS Nodeã«é¢ãããªãœãŒã¹ãå®çŸ©ããŠããŸãã Outpostsäžã§ã¯Auto Scale Groupã®å©çšãå¯èœãªãããã®èšå®ãå®çŸ©ããŠããŸããããã§ã®ç¹åŸŽã¯Auto Scale Groupã®åããŒãã«å¯Ÿã㊠user_data ã®äžã®åŠçã§CoIPãèšå®ããŠãããšããã§ããAuto Scale Groupã«ã¯èªåã§EIPãèšå®ãçŸç¶ã§ããªãããããã®èšå®ã«ããLocal Gatewayçµç±ã§ã€ã³ã¿ãŒãããããªã³ãã¬ç°å¢ãšéä¿¡ããããšãå¯èœãšãªããŸãã ãŸãã aws_auth ã§ã¯Bastionã®RoleãAPIãµãŒãã«ã¢ã¯ã»ã¹ããããã®èšå®ãã¡ã€ã«ãäœæããŠããŸãã security_group.tf resource "aws_security_group" "cluster" { vpc_id = aws_vpc.main.id name = "eks-cluster-security-group-$ { var.name } " tags = { Name = "eks-cluster-security-group-$ { var.name } " } ingress { from_port = 1025 to_port = 65535 protocol = "tcp" security_groups = [ aws_security_group.node.id ] } ingress { from_port = 443 to_port = 443 protocol = "tcp" security_groups = [ aws_security_group.node.id, aws_security_group.bastion.id ] } egress { from_port = 0 to_port = 0 protocol = "-1" cidr_blocks = [ "0.0.0.0/0" ] } } resource "aws_security_group" "node" { vpc_id = aws_vpc.main.id name = "eks-node-sg-$ { var.name } " tags = { Name = "eks-node-sg-$ { var.name } " } egress { from_port = 0 to_port = 0 protocol = "-1" cidr_blocks = [ "0.0.0.0/0" ] } } resource "aws_security_group_rule" "node_ingress_443" { security_group_id = aws_security_group.node.id type = "ingress" from_port = 443 to_port = 443 protocol = "tcp" source_security_group_id = aws_security_group.cluster.id } resource "aws_security_group_rule" "node_ingress_1025_65535" { security_group_id = aws_security_group.node.id type = "ingress" from_port = 1025 to_port = 65535 protocol = "tcp" source_security_group_id = aws_security_group.cluster.id } resource "aws_security_group_rule" "node_ingress_localgateway" { security_group_id = aws_security_group.node.id type = "ingress" from_port = 0 to_port = 0 protocol = "all" cidr_blocks = var.local_cidr_blocks } resource "aws_security_group_rule" "node_ingress_self" { security_group_id = aws_security_group.node.id type = "ingress" from_port = 0 to_port = 0 protocol = "all" self = true } resource "aws_security_group" "vpc_endpoint_ec2" { vpc_id = aws_vpc.main.id name = "vpc-endpoint-ec2-sg-$ { var.name } " tags = { Name = "vpc-endpoint-ec2-sg-$ { var.name } " } ingress { from_port = 0 to_port = 0 protocol = "all" security_groups = [ aws_security_group.node.id ] } egress { from_port = 0 to_port = 0 protocol = "-1" cidr_blocks = [ "0.0.0.0/0" ] } } resource "aws_security_group" "bastion" { vpc_id = aws_vpc.main.id name = "bastion-sg-$ { var.name } " tags = { Name = "bastion-sg-$ { var.name } " } # ãªã³ãã¬ç°å¢ããã®ã¿ã¢ã¯ã»ã¹å¯èœãªèšå®ã«ãã ingress { from_port = 22 to_port = 22 protocol = "tcp" cidr_blocks = var.local_cidr_blocks } egress { from_port = 0 to_port = 0 protocol = "-1" cidr_blocks = [ "0.0.0.0/0" ] } } security_group.tfã«ã¯Security Groupã«é¢ãããªãœãŒã¹ãå®çŸ©ããŸãã ãã¡ãã§ã¯ãEKSããªã³ãã¬ç°å¢ãšã®Security Groupã«ã€ããŠèšå®ãããŠããŸãã local_cidr_blocks ã«ã€ããŠã¯é©å®ããããã®ãªã³ãã¬ç°å¢ã«é©ããèšå®ãå¿
èŠãšãªããŸããEKSã«ã€ããŠã®è©³çŽ°ã¯æ¬¡ã®å
¬åŒããã¥ã¡ã³ããåç
§ããŠãã ããã Amazon EKS ã»ãã¥ãªãã£ã°ã«ãŒãã®èæ
®äºé
bastion.tf locals { # APIãµãŒããæäœããããã®kubectl, helmãªã©ã®åçš®ããŒã«ãã€ã³ã¹ããŒã«ãã bastion_user_data = <<EOF #!/bin/bash set -o xtrace # install kubectl curl --retry 180 --retry-delay 1 -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" chmod +x ./kubectl sudo mv ./kubectl /usr/local/bin/kubectl # install helm curl -sSL https: //raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash EOF } data "aws_ssm_parameter" "amzn2_ami" { name = "/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2" } data "aws_iam_policy_document" "assume_role" { statement { actions = [ "sts:AssumeRole" ] principals { type = "Service" identifiers = [ "ec2.amazonaws.com" ] } } } resource "aws_iam_role" "bastion" { name = "bastion-role-$ { var.name } " assume_role_policy = data.aws_iam_policy_document.assume_role.json } # APIãµãŒãèªèšŒã®ããã®IAM Policy resource "aws_iam_policy" "bastion" { name = "bastion-policy-$ { var.name } " policy = <<EOF { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "eks:UpdateClusterConfig", "eks:DescribeUpdate", "eks:DescribeCluster" ], "Resource": "$ { aws_eks_cluster.main.arn } " }, { "Effect": "Allow", "Action": [ "eks:ListClusters" ], "Resource": "*" } ] } EOF } resource "aws_iam_role_policy_attachment" "bastion" { role = aws_iam_role.bastion.name policy_arn = aws_iam_policy.bastion.arn } resource "aws_iam_instance_profile" "bastion" { role = aws_iam_role.bastion.name } resource "aws_instance" "bastion" { ami = data.aws_ssm_parameter.amzn2_ami.value subnet_id = aws_subnet.outposts_subnet01.id instance_type = var.bastion_instance_type key_name = aws_key_pair.key.key_name vpc_security_group_ids = [ aws_security_group.bastion.id ] iam_instance_profile = aws_iam_instance_profile.bastion.name user_data = local.bastion_user_data tags = { Name = "bastion-instance-$ { var.name } " } } resource "aws_eip_association" "bastion" { instance_id = aws_instance.bastion.id allocation_id = aws_eip.bastion.id } bastion.tfã«ã¯Bastion(èžã¿å°)ã«é¢ãããªãœãŒã¹ãå®çŸ©ããŸãã ãã¡ãã§ã¯ãBastionã«EKSã®APIãµãŒããæäœå¯èœãšããããã®IAM Roleãèšå®ããŠããããšãããããšæããŸãããŸãéä¿¡ãCoIPãèšå®ããŠãªã³ãã¬ç°å¢ããã®ã¿æ¥ç¶ã§ããããã«èšå®ããŸãã output.tf output "cluster" { value = aws_eks_cluster.main.name } output "alb_arn" { value = aws_iam_policy.aws_load_balancer_controller.arn } output "region" { value = var.region } output "ip_pool" { value = local.customer_owned_ipv4_pool } output "vpc_cidr" { value = aws_vpc.main.cidr_block } output "bastion_ip" { value = aws_eip.bastion.customer_owned_ip } output.tfã«ã¯k8sã¢ããªã±ãŒã·ã§ã³ã®ãããã€ã«äœ¿çšããå€ãå®çŸ©ããŸãã Terraform Apply äžèšã®tfãã¡ã€ã«ã次ã®ããã«applyããããšã§äžé£ã®ãªãœãŒã¹ã®èªåæ§ç¯ãå¯èœã§ãã terraform init terraform plan # dry run terraform apply Setup EKS åæã«ã¯ãAPIãµãŒãã«ã¯EKSã¯ã©ã¹ã¿äœæè
ã®ã¿ãã¢ã¯ã»ã¹å¯èœãªãã sshuttle ãšããsshã§Bastionçµç±ã®éä¿¡ãå¯èœãšããããŒã«ãçšããŸããaws-authãapplyããåŸã¯Bastionäžã§ã®æäœãå¯èœãšãªããŸãããŸãåãã« helm ãçšããŠEKSäžã§ALBå©çšããããã®OIDCãServiceAccountãªã©ã®èšå®ãããŠããµã³ãã«ã¢ããªã±ãŒã·ã§ã³game2048ããããã€ããŠã¿ãŸãã äºåã«äœ¿çšãã sshuttle , kubectl , helm ãªã©ã®ããŒã«ãã€ã³ã¹ããŒã«ããŸãã åäœç¢ºèªç°å¢: Apple M1 Max MacBookPro18,4 macOS 12.2.1 Darwin 21.3.0 brew install sshuttle brew install kubectl brew install helm brew tap weaveworks/tap brew install weaveworks/tap/eksctl CLUSTER="`terraform output -raw cluster`" ALB_ARN="`terraform output -raw alb_arn`" REGION="`terraform output -raw region`" IP_POOL="`terraform output -raw ip_pool`" BASTION_IP="`terraform output -raw bastion_ip`" VPC_CIDR="`terraform output -raw vpc_cidr`" sshuttle -r ec2-user@$BASTION_IP $VPC_CIDR & PID=$! aws eks --region $REGION update-kubeconfig --name $CLUSTER kubectl apply -f _aws-auth.yaml # oidc eksctl utils associate-iam-oidc-provider \ --region=ap-northeast-1 \ --cluster=$CLUSTER \ --region=$REGION \ --approve # attach policy eksctl create iamserviceaccount \ --cluster=$CLUSTER \ --namespace=kube-system \ --name=aws-load-balancer-controller \ --attach-policy-arn=$ALB_ARN \ --override-existing-serviceaccounts \ --approve # ssh bastion ssh ec2-user@$BASTION_IP # ALB helm install aws-load-balancer-controller eks/aws-load-balancer-controller \ -n kube-system \ --set clusterName=$CLUSTER \ --set serviceAccount.create=false \ --set image.repository=602401143452.dkr.ecr.ap-northeast-1.amazonaws.com/amazon/aws-load-balancer-controller \ --set serviceAccount.name=aws-load-balancer-controller # ALB Application game2048 curl -s https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.3.1/docs/examples/2048/2048_full.yaml | \ sed "51s|.*| alb.ingress.kubernetes.io/customer-owned-ipv4-pool: $IP_POOL|" > _2048_full.yaml kubectl apply -f _2048_full.yaml kill -9 $PID ingressã®èšå®ã®äžã« alb.ingress.kubernetes.io/customer-owned-ipv4-pool 3 ãèšå®ããããšã§ãªã³ãã¬ããã¢ã¯ã»ã¹å¯èœãªALBãäœæããããšãå¯èœãšãªããŸãã äžèšã®ããã«game-2048ãdeployããããšã§ä»¥äžã®ããã«ingressãç«ã¡äžãããŸãã衚瀺ãããURIã«ãªã³ãã¬ç°å¢ããã¢ã¯ã»ã¹ããããšã§ãµã³ãã«ã®ã¢ããªã±ãŒã·ã§ã³ã衚瀺ãããŸãã kubectl get ingress -n game-2048 NAME CLASS HOSTS ADDRESS PORTS AGE ingress-2048 <none> * k8s-game2048-ingress2-xxxxxxxxx-1234567890.ap-northeast-1.elb.amazonaws.com 80 4d10h 以äžã®ããã«ãµã³ãã«ã¢ããªã±ãŒã·ã§ã³ãå®è¡ã§ããŸããã ãŸãšã æ¬èšäºã§ã¯ãTerraformãçšããŠOutpostsäžã§ãªã³ãã¬ç°å¢ããã®ã¿ç®¡çã»ã¢ã¯ã»ã¹å¯èœãªPrivate EKSãæ§ç¯ããæ¹æ³ã玹ä»ããŸããã éçºé¢ã§ã¯ããªã³ãã¬ç°å¢ã§ãIaCãçšããŠãããŒãžãk8sãå©çšããããšã倧ããªã¡ãªããããšæããŸããå©çšé¢ã§ãã»ãã¥ã¢ãªå¯çšæ§ã®é«ãã¢ããªã±ãŒã·ã§ã³ãæ§ç¯ããããšãå¯èœãšãªãããéåžžã«æçšã§ãããšæããŸãã https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/subnet ↩ https://docs.aws.amazon.com/autoscaling/ec2/userguide/what-is-amazon-ec2-auto-scaling.html ↩ https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.1/guide/ingress/annotations/#customer-owned-ipv4-pool ↩