From d22c2adbe3588e17b03225718b526c03aad3b2f8 Mon Sep 17 00:00:00 2001 From: ashgit301 Date: Mon, 14 Jun 2021 16:31:24 +0530 Subject: [PATCH] microk8s+terraform Change-Id: I7658824883a960abed0925e0eda6b59f6a13411e Signed-off-by: ashgit301 --- src/foundation/microk8s/main.tf | 128 ++++++++++++++++++++++++++ src/foundation/microk8s/microk8s.join_token | 5 + src/foundation/microk8s/readme | 91 ++++++++++++++++++ src/foundation/microk8s/token | 7 ++ src/foundation/microk8s/variable.tf | 35 +++++++ src/foundation/microk8s/worker_user_data.tmpl | 10 ++ 6 files changed, 276 insertions(+) create mode 100644 src/foundation/microk8s/main.tf create mode 100644 src/foundation/microk8s/microk8s.join_token create mode 100644 src/foundation/microk8s/readme create mode 100644 src/foundation/microk8s/token create mode 100644 src/foundation/microk8s/variable.tf create mode 100644 src/foundation/microk8s/worker_user_data.tmpl diff --git a/src/foundation/microk8s/main.tf b/src/foundation/microk8s/main.tf new file mode 100644 index 0000000..ea33da5 --- /dev/null +++ b/src/foundation/microk8s/main.tf @@ -0,0 +1,128 @@ +provider "aws" { + region = var.aws_region + access_key = var.access_key + secret_key = var.secret_key + +} + +resource "aws_instance" "master" { + ami = var.aws_ami + instance_type = var.aws_instance + vpc_security_group_ids = [aws_security_group.instance.id] + subnet_id = var.aws_subnet_id + user_data = <<-EOF + #!/bin/bash + sudo su + apt update -y >> microk8s_install.log + apt install snapd -y >> microk8s_install.log + snap install core >> microk8s_install.log + export PATH=$PATH:/snap/bin + snap install microk8s --classic >> microk8s_install.log + microk8s status --wait-ready + microk8s enable dns >> microk8s_install.log + microk8s add-node > microk8s.join_token + microk8s config > configFile + EOF + key_name = "terraform" + tags = { + Name = "master" + } + provisioner "remote-exec" { + inline = ["until [ -f /microk8s.join_token ]; do sleep 5; done; cat /microk8s.join_token", + "sudo sed -i 's/#MOREIPS/IP.7 = ${self.public_ip}\\n#MOREIPS/g' /var/snap/microk8s/current/certs/csr.conf.template", + "sudo sleep 1m", + "sudo microk8s stop", + "sudo microk8s start" + ] + } + + connection { + host = self.public_ip + type = "ssh" + user = "ubuntu" + password = "" + private_key = "${file("terraform.pem")}" + } + + provisioner "local-exec" { + command = <> ~/.ssh/known_hosts + scp -i terraform.pem ubuntu@${self.public_dns}:/microk8s.join_token . + tail -n1 microk8s.join_token >> token + scp -i terraform.pem ubuntu@${self.public_dns}:/configFile . + EOT + } + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_security_group" "instance" { + name = "master_microk8s" + vpc_id = var.vpc_id + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + +} + +data "local_file" "tokenContent" { + filename = "token" + depends_on = [ + aws_instance.master, + ] +} + + +locals { + public_dns = aws_instance.master.public_dns + join = data.local_file.tokenContent.content +} + + +resource "aws_instance" "worker" { + ami = var.aws_ami + instance_type = var.aws_instance + vpc_security_group_ids = [aws_security_group.instance.id] + subnet_id = var.aws_subnet_id + user_data = templatefile("worker_user_data.tmpl", { token = local.join } ) + key_name = "terraform" + tags = { + Name = "worker" + } + + provisioner "remote-exec" { + inline = ["until [ -f /microk8s.complete ]; do sleep 5; done"] + } + + connection { + host = self.public_ip + type = "ssh" + user = "ubuntu" + password = "" + private_key = "${file("terraform.pem")}" + } + lifecycle { + create_before_destroy = true + } + + depends_on = [ + aws_instance.master, + ] +} + + +output "master_ip" { + value = aws_instance.master.public_ip +} + diff --git a/src/foundation/microk8s/microk8s.join_token b/src/foundation/microk8s/microk8s.join_token new file mode 100644 index 0000000..4c9e73f --- /dev/null +++ b/src/foundation/microk8s/microk8s.join_token @@ -0,0 +1,5 @@ +From the node you wish to join to this cluster, run the following: +microk8s join 172.31.10.54:25000/f9b66e02527274bd62996de9e04da30b + +If the node you are adding is not reachable through the default interface you can use one of the following: + microk8s join 172.31.10.54:25000/f9b66e02527274bd62996de9e04da30b diff --git a/src/foundation/microk8s/readme b/src/foundation/microk8s/readme new file mode 100644 index 0000000..517da56 --- /dev/null +++ b/src/foundation/microk8s/readme @@ -0,0 +1,91 @@ +Terraform + microk8s +-------------------- +The purpose of this terraform template is to provision multi-node kubernetes cluster on AWS using microk8s. MicroK8s offers a light weight kubernetes environment for edge use cases. + +Pre-requisite +------------- +1. Install terraform - https://www.terraform.io/downloads.html + + (a)Downlaod the zip file based on the server type. + (b)Unzip the file to get the terraform binary. + (c)Currently supported ubuntu version is 18.04 + +2. IAM Access Keys - Permissions required for running the template - AmazonEC2FullAccess +3. PEM file for the AWS Key used in the terraform template +NOTE : Replace fields in the variable.tf file with your corresponding values + +In order for Terraform to be able to create resources in your AWS account, you will need to configure the AWS credentials. One of the easiest ofwhich is to set the following environment variables: + +export AWS_ACCESS_KEY_ID=(your access key id) +export AWS_SECRET_ACCESS_KEY=(your secret access key) + +The credentials can also be set in the variable.tf file. + +variable "access_key" { + description = "access_key" + default = +} + +variable "secret_key" { + description = "secret_key" + default = +} + + +Terraform template +------------------ +The template contains main.tf file, variable.tf file, pem file (add your pem file here) and worker_user_data.tmpl +You can move the pem file to the the directory where this template resides or you can change the location of the pem file in the main.tf file. + +Master's main.tf file +-------------------- +The first step to using Terraform is typically to configure the provider(s) you want to use. +This tells Terraform that you are going to be using the AWS provider and that you wish to deploy your infrastructure in the us-east-2 region. + + provider "aws" { + region = var.aws_region + } + +The user_data installs the microk8s inside the EC2 instance. + + #!/bin/bash + sudo su + apt update -y >> microk8s_install.log + apt install snapd -y >> microk8s_install.log + snap install core >> microk8s_install.log + export PATH=$PATH:/snap/bin + snap install microk8s --classic >> microk8s_install.log + microk8s status --wait-ready + microk8s enable dns >> microk8s_install.log + microk8s add-node > microk8s.join_token + microk8s config > configFile + +Since terraform does not wait until the user_data is executed, we exec into the instace by using the 'remote-exec' type provisioner and add the following script. This script will make terraform wait util microk8s.join-token file is created. + + provisioner "remote-exec" { + inline = ["until [ -f /microk8s.join_token ]; do sleep 5; done; cat /microk8s.join_token"] + } + +For testing purposes, we create an 'ALLOW ALL' ingress and egress rule security group. + +Variables.tf file +---------------- +The provider and the resource blocks in the main.tf file can be configured by changing the values in variables.tf file. +For example, if you want to change the aws_instace type from t2.small to t2.micro, replace the value here in this block. + variable "aws_instance" { + type = string + description = "instance_type" + default = "t2.small" + } +Other resource specific values like aws_region, aws_ami, vpc and the subenet can also be changed the same way by editing the variable.tf file. + +Apply terraform +--------------- + +To create a master node with microk8s, run the following commands. +terraform init +terraform plan +terraform apply + +Once the worked nodes are created, it will be connected to the master. A multi-node k8s cluster will be provisioned with calico CNI. + diff --git a/src/foundation/microk8s/token b/src/foundation/microk8s/token new file mode 100644 index 0000000..328ed78 --- /dev/null +++ b/src/foundation/microk8s/token @@ -0,0 +1,7 @@ + microk8s join 172.31.21.68:25000/7bbce642aec6aafd22028238f1114c88 + microk8s join 172.31.18.98:25000/629d7bf742ec927cc21963b2ff404def + microk8s join 172.31.3.108:25000/48fb4f98b7427abd52c45f522defc268 + microk8s join 172.31.30.69:25000/ce3caa0dcffb57b71741dd4f95b8306d + microk8s join 172.31.22.56:25000/d60f8e7fb55b5e478a634095fb1c270f + microk8s join 172.31.16.138:25000/297050fa5e93c71e9fda831ba64b99f9 + microk8s join 172.31.10.54:25000/f9b66e02527274bd62996de9e04da30b diff --git a/src/foundation/microk8s/variable.tf b/src/foundation/microk8s/variable.tf new file mode 100644 index 0000000..6f8a94d --- /dev/null +++ b/src/foundation/microk8s/variable.tf @@ -0,0 +1,35 @@ +variable "aws_region" { + description = "aws_region" + default = "us-east-2" +} + +variable "aws_instance" { + description = "instance_type" + default = "t2.small" +} + +variable "aws_ami" { + description = "aws_ami" + default = "ami-026141f3d5c6d2d0c" +} + +variable "aws_subnet_id" { + description = "subnet_id" + default = "" +} + +variable "vpc_id" { + description = "vpc_id" + default = "" +} + +variable "access_key" { + description = "access_key" + default = "" +} + +variable "secret_key" { + description = "secret_key" + default = "" +} + diff --git a/src/foundation/microk8s/worker_user_data.tmpl b/src/foundation/microk8s/worker_user_data.tmpl new file mode 100644 index 0000000..4878132 --- /dev/null +++ b/src/foundation/microk8s/worker_user_data.tmpl @@ -0,0 +1,10 @@ +#!/bin/bash + +sudo apt update -y >> microk8s_install.log +sudo apt install snapd -y >> microk8s_install.log +sudo snap install microk8s --classic >> microk8s_install.log +sudo microk8s enable dns >> microk8s_install.log +sudo ${token} >> microk8s_install.log +sudo touch microk8s.complete + + -- 2.16.6