14 bpav1alpha1 "github.com/bpa-operator/pkg/apis/bpa/v1alpha1"
15 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
16 corev1 "k8s.io/api/core/v1"
17 batchv1 "k8s.io/api/batch/v1"
18 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
19 "k8s.io/apimachinery/pkg/runtime/schema"
20 "k8s.io/apimachinery/pkg/api/errors"
21 "k8s.io/apimachinery/pkg/runtime"
22 "k8s.io/client-go/dynamic"
23 "k8s.io/client-go/rest"
25 "k8s.io/client-go/kubernetes"
26 "sigs.k8s.io/controller-runtime/pkg/client"
27 "sigs.k8s.io/controller-runtime/pkg/client/config"
28 "sigs.k8s.io/controller-runtime/pkg/controller"
29 "sigs.k8s.io/controller-runtime/pkg/handler"
30 "sigs.k8s.io/controller-runtime/pkg/manager"
31 "sigs.k8s.io/controller-runtime/pkg/reconcile"
32 logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
33 "sigs.k8s.io/controller-runtime/pkg/source"
35 "golang.org/x/crypto/ssh"
38 var log = logf.Log.WithName("controller_provisioning")
41 * USER ACTION REQUIRED: This is a scaffold file intended for the user to modify with their own Controller
42 * business logic. Delete these comments after modifying this file.*
45 // Add creates a new Provisioning Controller and adds it to the Manager. The Manager will set fields on the Controller
46 // and Start it when the Manager is Started.
47 func Add(mgr manager.Manager) error {
48 return add(mgr, newReconciler(mgr))
51 // newReconciler returns a new reconcile.Reconciler
52 func newReconciler(mgr manager.Manager) reconcile.Reconciler {
53 return &ReconcileProvisioning{client: mgr.GetClient(), scheme: mgr.GetScheme()}
56 // add adds a new Controller to mgr with r as the reconcile.Reconciler
57 func add(mgr manager.Manager, r reconcile.Reconciler) error {
58 // Create a new controller
59 c, err := controller.New("provisioning-controller", mgr, controller.Options{Reconciler: r})
64 // Watch for changes to primary resource Provisioning
65 err = c.Watch(&source.Kind{Type: &bpav1alpha1.Provisioning{}}, &handler.EnqueueRequestForObject{})
70 // Watch for changes to resource configmap created as a consequence of the provisioning CR
71 err = c.Watch(&source.Kind{Type: &corev1.ConfigMap{}}, &handler.EnqueueRequestForOwner{
73 OwnerType: &bpav1alpha1.Provisioning{},
80 //Watch for changes to job resource also created as a consequence of the provisioning CR
81 err = c.Watch(&source.Kind{Type: &batchv1.Job{}}, &handler.EnqueueRequestForOwner{
83 OwnerType: &bpav1alpha1.Provisioning{},
91 // Watch for changes to resource software CR
92 err = c.Watch(&source.Kind{Type: &bpav1alpha1.Software{}}, &handler.EnqueueRequestForObject{})
102 // blank assignment to verify that ReconcileProvisioning implements reconcile.Reconciler
103 var _ reconcile.Reconciler = &ReconcileProvisioning{}
105 // ReconcileProvisioning reconciles a Provisioning object
106 type ReconcileProvisioning struct {
107 // This client, initialized using mgr.Client() above, is a split client
108 // that reads objects from the cache and writes to the apiserver
110 scheme *runtime.Scheme
113 // Reconcile reads that state of the cluster for a Provisioning object and makes changes based on the state read
114 // and what is in the Provisioning.Spec
115 // TODO(user): Modify this Reconcile function to implement your Controller logic. This example creates
116 // a Pod as an example
118 // The Controller will requeue the Request to be processed again if the returned error is non-nil or
119 // Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
120 func (r *ReconcileProvisioning) Reconcile(request reconcile.Request) (reconcile.Result, error) {
121 reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name)
122 //reqLogger.Info("Reconciling Provisioning")
124 reqLogger.Info("Reconciling Custom Resource")
128 // Fetch the Provisioning instance
129 provisioningInstance := &bpav1alpha1.Provisioning{}
130 softwareInstance := &bpav1alpha1.Software{}
131 err := r.client.Get(context.TODO(), request.NamespacedName, provisioningInstance)
132 provisioningCreated := true
135 //Check if its a Software Instance
136 err = r.client.Get(context.TODO(), request.NamespacedName, softwareInstance)
138 if errors.IsNotFound(err) {
139 // Request object not found, could have been deleted after reconcile request.
140 // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers.
141 // Return and don't requeue
142 return reconcile.Result{}, nil
145 // Error reading the object - requeue the request.
146 return reconcile.Result{}, err
149 //No error occured and so a Software CR was created not a Provisoning CR
150 provisioningCreated = false
154 masterTag := "MASTER_"
155 workerTag := "WORKER_"
157 config, err := config.GetConfig()
159 fmt.Printf("Could not get kube config, Error: %v\n", err)
160 return reconcile.Result{}, err
163 clientset, err := kubernetes.NewForConfig(config)
165 fmt.Printf("Could not create clientset, Error: %v\n", err)
166 return reconcile.Result{}, err
168 if provisioningCreated {
170 ///////////////////////////////////////////////////////////////////////////////////////////////
171 //////////////// Provisioning CR was created so install KUD /////////////////
172 //////////////////////////////////////////////////////////////////////////////////////////////
173 clusterName := provisioningInstance.Labels["cluster"]
174 mastersList := provisioningInstance.Spec.Masters
175 workersList := provisioningInstance.Spec.Workers
176 dhcpLeaseFile := provisioningInstance.Spec.DHCPleaseFile
177 kudInstallerScript := provisioningInstance.Spec.KUDInstaller
178 multiClusterDir := provisioningInstance.Spec.MultiClusterPath
181 bareMetalHostList, _ := listBareMetalHosts(config)
185 var masterString string
186 var workerString string
188 defaultDHCPFile := "/var/lib/dhcp/dhcpd.leases"
189 defaultKUDInstallerPath := "/multicloud-k8s/kud/hosting_providers/vagrant"
190 defaultMultiClusterDir := "/multi-cluster"
192 //Give Default values for paths if no path is given in the CR
193 if dhcpLeaseFile == "" {
194 dhcpLeaseFile = defaultDHCPFile
197 if kudInstallerScript == "" {
198 kudInstallerScript = defaultKUDInstallerPath
201 if multiClusterDir == "" {
202 multiClusterDir = defaultMultiClusterDir
205 //Create Directory for the specific cluster
206 clusterDir := multiClusterDir + "/" + clusterName
207 os.MkdirAll(clusterDir, os.ModePerm)
209 //Create Maps to be used for cluster ip address to label configmap
210 clusterLabel := make(map[string]string)
211 clusterLabel["cluster"] = clusterName
212 clusterData := make(map[string]string)
216 //Iterate through mastersList and get all the mac addresses and IP addresses
217 for _, masterMap := range mastersList {
219 for masterLabel, master := range masterMap {
220 masterMAC := master.MACaddress
223 err = fmt.Errorf("MAC address for masterNode %s not provided\n", masterLabel)
224 return reconcile.Result{}, err
226 containsMac, bmhCR := checkMACaddress(bareMetalHostList, masterMAC)
228 fmt.Printf("BareMetalHost CR %s has NIC with MAC Address %s\n", bmhCR, masterMAC)
230 //Get IP address of master
231 hostIPaddress, err := getHostIPaddress(masterMAC, dhcpLeaseFile )
232 if err != nil || hostIPaddress == ""{
233 err = fmt.Errorf("IP address not found for host with MAC address %s \n", masterMAC)
234 return reconcile.Result{}, err
237 allString += masterLabel + " ansible_ssh_host=" + hostIPaddress + " ansible_ssh_port=22" + "\n"
238 masterString += masterLabel + "\n"
239 clusterData[masterTag + masterLabel] = hostIPaddress
241 fmt.Printf("%s : %s \n", hostIPaddress, masterMAC)
243 if len(workersList) != 0 {
245 //Iterate through workersList and get all the mac addresses
246 for _, workerMap := range workersList {
248 //Get worker labels from the workermap
249 for workerLabel, worker := range workerMap {
251 //Check if workerString already contains worker label
252 containsWorkerLabel := strings.Contains(workerString, workerLabel)
253 workerMAC := worker.MACaddress
255 //Error occurs if the same label is given to different hosts (assumption,
256 //each MAC address represents a unique host
257 if workerLabel == masterLabel && workerMAC != masterMAC && workerMAC != "" {
258 if containsWorkerLabel {
259 strings.ReplaceAll(workerString, workerLabel, "")
261 err = fmt.Errorf(`A node with label %s already exists, modify resource and assign a
262 different label to node with MACAddress %s`, workerLabel, workerMAC)
263 return reconcile.Result{}, err
265 //same node performs worker and master roles
266 } else if workerLabel == masterLabel && !containsWorkerLabel {
267 workerString += workerLabel + "\n"
269 //Add host to ip address config map with worker tag
270 hostIPaddress = clusterData[masterTag + masterLabel]
271 clusterData[workerTag + masterLabel] = hostIPaddress
273 //Error occurs if the same node is given different labels
274 } else if workerLabel != masterLabel && workerMAC == masterMAC {
275 if containsWorkerLabel {
276 strings.ReplaceAll(workerString, workerLabel, "")
278 err = fmt.Errorf(`A node with label %s already exists, modify resource and assign a
279 different label to node with MACAddress %s`, workerLabel, workerMAC)
280 return reconcile.Result{}, err
282 //worker node is different from any master node and it has not been added to the worker list
283 } else if workerLabel != masterLabel && !containsWorkerLabel {
285 // Error occurs if MAC address not provided for worker node not matching master
287 err = fmt.Errorf("MAC address for worker %s not provided", workerLabel)
288 return reconcile.Result{}, err
291 containsMac, bmhCR := checkMACaddress(bareMetalHostList, workerMAC)
293 fmt.Printf("Host %s matches that macAddress\n", bmhCR)
295 //Get IP address of worker
296 hostIPaddress, err := getHostIPaddress(workerMAC, dhcpLeaseFile )
298 fmt.Errorf("IP address not found for host with MAC address %s \n", workerMAC)
299 return reconcile.Result{}, err
301 fmt.Printf("%s : %s \n", hostIPaddress, workerMAC)
305 allString += workerLabel + " ansible_ssh_host=" + hostIPaddress + " ansible_ssh_port=22" + "\n"
306 workerString += workerLabel + "\n"
307 clusterData[workerTag + workerLabel] = hostIPaddress
309 //No host found that matches the worker MAC
312 err = fmt.Errorf("Host with MAC Address %s not found\n", workerMAC)
313 return reconcile.Result{}, err
319 //No worker node specified, add master as worker node
320 } else if len(workersList) == 0 && !strings.Contains(workerString, masterLabel) {
321 workerString += masterLabel + "\n"
323 //Add host to ip address config map with worker tag
324 hostIPaddress = clusterData[masterTag + masterLabel]
325 clusterData[workerTag + masterLabel] = hostIPaddress
328 //No host matching master MAC found
330 err = fmt.Errorf("Host with MAC Address %s not found\n", masterMAC)
331 return reconcile.Result{}, err
336 //Create host.ini file
337 //iniHostFilePath := kudInstallerScript + "/inventory/hosts.ini"
338 iniHostFilePath := clusterDir + "/hosts.ini"
339 newFile, err := os.Create(iniHostFilePath)
340 defer newFile.Close()
344 fmt.Printf("Error occured while creating file \n %v", err)
345 return reconcile.Result{}, err
348 hostFile, err := ini.Load(iniHostFilePath)
350 fmt.Printf("Error occured while Loading file \n %v", err)
351 return reconcile.Result{}, err
354 _, err = hostFile.NewRawSection("all", allString)
356 fmt.Printf("Error occured while creating section \n %v", err)
357 return reconcile.Result{}, err
359 _, err = hostFile.NewRawSection("kube-master", masterString)
361 fmt.Printf("Error occured while creating section \n %v", err)
362 return reconcile.Result{}, err
365 _, err = hostFile.NewRawSection("kube-node", workerString)
367 fmt.Printf("Error occured while creating section \n %v", err)
368 return reconcile.Result{}, err
371 _, err = hostFile.NewRawSection("etcd", masterString)
373 fmt.Printf("Error occured while creating section \n %v", err)
374 return reconcile.Result{}, err
377 _, err = hostFile.NewRawSection("k8s-cluster:children", "kube-node\n" + "kube-master")
379 fmt.Printf("Error occured while creating section \n %v", err)
380 return reconcile.Result{}, err
384 //Create host.ini file for KUD
385 hostFile.SaveTo(iniHostFilePath)
388 err = createKUDinstallerJob(clusterName, request.Namespace, clusterLabel, clientset)
390 fmt.Printf("Error occured while creating KUD Installer job for cluster %v\n ERROR: %v", clusterName, err)
391 return reconcile.Result{}, err
394 //Start separate thread to keep checking job status, Create an IP address configmap
395 //for cluster if KUD is successfully installed
396 go checkJob(clusterName, request.Namespace, clusterData, clusterLabel, clientset)
398 return reconcile.Result{}, nil
404 ///////////////////////////////////////////////////////////////////////////////////////////////
405 //////////////// Software CR was created so install software /////////////////
406 //////////////////////////////////////////////////////////////////////////////////////////////
407 softwareClusterName, masterSoftwareList, workerSoftwareList := getSoftwareList(softwareInstance)
408 defaultSSHPrivateKey := "/root/.ssh/id_rsa"
410 //Get IP address configmap for the cluster
411 clusterConfigMapData, err := getConfigMapData(request.Namespace, softwareClusterName, clientset)
413 fmt.Printf("Error occured while retrieving IP address Data for cluster %s, ERROR: %v\n", softwareClusterName, err)
414 return reconcile.Result{}, err
417 for hostLabel, ipAddress := range clusterConfigMapData {
419 if strings.Contains(hostLabel, masterTag) {
420 // Its a master node, install master software
421 err = softwareInstaller(ipAddress, defaultSSHPrivateKey, masterSoftwareList)
423 fmt.Printf("Error occured while installing master software in host %s, ERROR: %v\n", hostLabel, err)
425 } else if strings.Contains(hostLabel, workerTag) {
426 // Its a worker node, install worker software
427 err = softwareInstaller(ipAddress, defaultSSHPrivateKey, workerSoftwareList)
429 fmt.Printf("Error occured while installing worker software in host %s, ERROR: %v\n", hostLabel, err)
436 return reconcile.Result{}, nil
440 //Function to Get List containing baremetal hosts
441 func listBareMetalHosts(config *rest.Config) (*unstructured.UnstructuredList, error) {
443 //Create Dynamic Client for BareMetalHost CRD
444 bmhDynamicClient, err := dynamic.NewForConfig(config)
447 fmt.Println("Could not create dynamic client for bareMetalHosts, Error: %v\n", err)
448 return &unstructured.UnstructuredList{}, err
451 //Create GVR representing a BareMetalHost CR
452 bmhGVR := schema.GroupVersionResource{
455 Resource: "baremetalhosts",
458 //Get List containing all BareMetalHosts CRs
459 bareMetalHosts, err := bmhDynamicClient.Resource(bmhGVR).List(metav1.ListOptions{})
461 fmt.Println("Error occured, cannot get BareMetalHosts list, Error: %v\n", err)
462 return &unstructured.UnstructuredList{}, err
465 return bareMetalHosts, nil
469 //Function to check if BareMetalHost containing MAC address exist
470 func checkMACaddress(bareMetalHostList *unstructured.UnstructuredList, macAddress string) (bool, string) {
472 //Convert macAddress to byte array for comparison
473 macAddressByte := []byte(macAddress)
476 for _, bareMetalHost := range bareMetalHostList.Items {
477 bmhJson, _ := bareMetalHost.MarshalJSON()
479 macBool = bytes.Contains(bmhJson, macAddressByte)
481 return macBool, bareMetalHost.GetName()
491 //Function to get the IP address of a host from the DHCP file
492 func getHostIPaddress(macAddress string, dhcpLeaseFilePath string ) (string, error) {
494 //Read the dhcp lease file
495 dhcpFile, err := ioutil.ReadFile(dhcpLeaseFilePath)
497 fmt.Println("Failed to read lease file\n")
501 dhcpLeases := string(dhcpFile)
503 //Regex to use to search dhcpLeases
504 reg := "lease.*{|ethernet.*|\n. binding state.*"
505 re, err := regexp.Compile(reg)
507 fmt.Println("Could not create Regexp object, Error %v occured\n", err)
511 //Get String containing leased Ip addresses and Corressponding MAC addresses
512 out := re.FindAllString(dhcpLeases, -1)
513 outString := strings.Join(out, " ")
514 stringReplacer := strings.NewReplacer("lease", "", "ethernet ", "", ";", "",
515 " binding state", "", "{", "")
516 replaced := stringReplacer.Replace(outString)
517 ipMacList := strings.Fields(replaced)
520 //Get IP addresses corresponding to Input MAC Address
521 for idx := len(ipMacList)-1 ; idx >= 0; idx -- {
522 item := ipMacList[idx]
523 if item == macAddress {
525 leaseState := ipMacList[idx -1]
526 if leaseState != "active" {
527 err := fmt.Errorf("No active ip address lease found for MAC address %s \n", macAddress)
528 fmt.Printf("%v\n", err)
531 ipAdd := ipMacList[idx - 2]
539 //Function to create configmap
540 func createConfigMap(data, labels map[string]string, namespace string, clientset *kubernetes.Clientset) error{
542 configmapClient := clientset.CoreV1().ConfigMaps(namespace)
544 configmap := &corev1.ConfigMap{
546 ObjectMeta: metav1.ObjectMeta{
547 Name: labels["cluster"] + "-configmap",
554 _, err := configmapClient.Create(configmap)
563 //Function to get configmap Data
564 func getConfigMapData(namespace, clusterName string, clientset *kubernetes.Clientset) (map[string]string, error) {
566 configmapClient := clientset.CoreV1().ConfigMaps(namespace)
567 configmapName := clusterName + "-configmap"
568 clusterConfigmap, err := configmapClient.Get(configmapName, metav1.GetOptions{})
573 configmapData := clusterConfigmap.Data
574 return configmapData, nil
578 //Function to create job for KUD installation
579 func createKUDinstallerJob(clusterName, namespace string, labels map[string]string, clientset *kubernetes.Clientset) error{
581 var backOffLimit int32 = 0
582 var privi bool = true
585 jobClient := clientset.BatchV1().Jobs("default")
589 ObjectMeta: metav1.ObjectMeta{
590 Name: "kud-" + clusterName,
593 Spec: batchv1.JobSpec{
594 Template: corev1.PodTemplateSpec{
595 ObjectMeta: metav1.ObjectMeta{
600 Spec: corev1.PodSpec{
602 Containers: []corev1.Container{{
604 Image: "github.com/onap/multicloud-k8s:latest",
605 ImagePullPolicy: "IfNotPresent",
606 VolumeMounts: []corev1.VolumeMount{{
607 Name: "multi-cluster",
608 MountPath: "/opt/kud/multi-cluster",
611 Name: "secret-volume",
616 Command: []string{"/bin/sh","-c"},
617 Args: []string{"cp -r /.ssh /root/; chmod -R 600 /root/.ssh; ./installer --cluster " + clusterName},
618 SecurityContext: &corev1.SecurityContext{
624 Volumes: []corev1.Volume{{
625 Name: "multi-cluster",
626 VolumeSource: corev1.VolumeSource{
627 HostPath: &corev1.HostPathVolumeSource{
628 Path : "/opt/kud/multi-cluster",
631 Name: "secret-volume",
632 VolumeSource: corev1.VolumeSource{
633 Secret: &corev1.SecretVolumeSource{
634 SecretName: "ssh-key-secret",
638 RestartPolicy: "Never",
642 BackoffLimit : &backOffLimit,
646 _, err := jobClient.Create(job)
648 fmt.Printf("ERROR occured while creating job to install KUD\n ERROR:%v", err)
655 //Function to Check if job succeeded
656 func checkJob(clusterName, namespace string, data, labels map[string]string, clientset *kubernetes.Clientset) {
658 fmt.Printf("\nChecking job status for cluster %s\n", clusterName)
659 jobName := "kud-" + clusterName
660 jobClient := clientset.BatchV1().Jobs(namespace)
663 time.Sleep(2 * time.Second)
665 job, err := jobClient.Get(jobName, metav1.GetOptions{})
667 fmt.Printf("ERROR: %v occured while retrieving job: %s", err, jobName)
670 jobSucceeded := job.Status.Succeeded
671 jobFailed := job.Status.Failed
673 if jobSucceeded == 1 {
674 fmt.Printf("\n Job succeeded, KUD successfully installed in Cluster %s\n", clusterName)
676 //KUD was installed successfully create configmap to store IP address info for the cluster
677 err = createConfigMap(data, labels, namespace, clientset)
679 fmt.Printf("Error occured while creating Ip address configmap for cluster %v\n ERROR: %v", clusterName, err)
686 fmt.Printf("\n Job Failed, KUD not installed in Cluster %s, check pod logs\n", clusterName)
694 //Function to get software list from software CR
695 func getSoftwareList(softwareCR *bpav1alpha1.Software) (string, []interface{}, []interface{}) {
697 CRclusterName := softwareCR.GetLabels()["cluster"]
699 masterSofwareList := softwareCR.Spec.MasterSoftware
700 workerSoftwareList := softwareCR.Spec.WorkerSoftware
702 return CRclusterName, masterSofwareList, workerSoftwareList
705 //Function to install software in clusterHosts
706 func softwareInstaller(ipAddress, sshPrivateKey string, softwareList []interface{}) error {
708 var installString string
709 for _, software := range softwareList {
711 switch t := software.(type){
713 installString += software.(string) + " "
715 softwareMap, errBool := software.(map[string]interface{})
717 fmt.Printf("Error occured, cannot install software %v\n", software)
719 for softwareName, versionMap := range softwareMap {
721 versionMAP, _ := versionMap.(map[string]interface{})
722 version := versionMAP["version"].(string)
723 installString += softwareName + "=" + version + " "
726 fmt.Printf("invalid format %v\n", t)
731 err := sshInstaller(installString, sshPrivateKey, ipAddress)
739 //Function to Run Installation commands via ssh
740 func sshInstaller(softwareString, sshPrivateKey, ipAddress string) error {
742 buffer, err := ioutil.ReadFile(sshPrivateKey)
747 key, err := ssh.ParsePrivateKey(buffer)
752 sshConfig := &ssh.ClientConfig{
754 Auth: []ssh.AuthMethod{
758 HostKeyCallback: ssh.InsecureIgnoreHostKey(),
761 client, err := ssh.Dial("tcp", ipAddress + ":22", sshConfig)
766 session, err := client.NewSession()
771 defer session.Close()
774 cmd := "sudo apt-get update && apt-get install " + softwareString + "-y"
775 err = session.Start(cmd)