cmd/bpa-operator/pkg/controller/provisioning/provisioning_controller.go

   1 package provisioning
   2
   3 import (
   4         "context"
   5         "os"
   6         "fmt"
   7         "time"
   8         "bytes"
   9         "regexp"
  10         "strings"
  11         "io/ioutil"
  12         "encoding/json"
  13
  14         bpav1alpha1 "github.com/bpa-operator/pkg/apis/bpa/v1alpha1"
  15         metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  16         corev1 "k8s.io/api/core/v1"
  17         batchv1 "k8s.io/api/batch/v1"
  18         "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
  19         "k8s.io/apimachinery/pkg/runtime/schema"
  20         "k8s.io/apimachinery/pkg/api/errors"
  21         "k8s.io/apimachinery/pkg/runtime"
  22         "k8s.io/client-go/dynamic"
  23
  24         "k8s.io/client-go/kubernetes"
  25         "sigs.k8s.io/controller-runtime/pkg/client"
  26         "sigs.k8s.io/controller-runtime/pkg/client/config"
  27         "sigs.k8s.io/controller-runtime/pkg/controller"
  28         "sigs.k8s.io/controller-runtime/pkg/handler"
  29         "sigs.k8s.io/controller-runtime/pkg/manager"
  30         "sigs.k8s.io/controller-runtime/pkg/reconcile"
  31         logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
  32         "sigs.k8s.io/controller-runtime/pkg/source"
  33         "gopkg.in/ini.v1"
  34         "golang.org/x/crypto/ssh"
  35 )
  36
  37 type VirtletVM struct {
  38         IPaddress string
  39         MACaddress string
  40 }
  41
  42 type NetworksStatus struct {
  43         Name string `json:"name,omitempty"`
  44         Interface string `json:"interface,omitempty"`
  45         Ips []string `json:"ips,omitempty"`
  46         Mac string `json:"mac,omitempty"`
  47         Default bool `json:"default,omitempty"`
  48         Dns interface{} `json:"dns,omitempty"`
  49 }
  50
  51 var log = logf.Log.WithName("controller_provisioning")
  52
  53 /**
  54 * USER ACTION REQUIRED: This is a scaffold file intended for the user to modify with their own Controller
  55 * business logic.  Delete these comments after modifying this file.*
  56  */
  57
  58 // Add creates a new Provisioning Controller and adds it to the Manager. The Manager will set fields on the Controller
  59 // and Start it when the Manager is Started.
  60 func Add(mgr manager.Manager) error {
  61         return add(mgr, newReconciler(mgr))
  62 }
  63
  64 // newReconciler returns a new reconcile.Reconciler
  65 func newReconciler(mgr manager.Manager) reconcile.Reconciler {
  66
  67         config, err :=  config.GetConfig()
  68         if err != nil {
  69            fmt.Printf("Could not get kube config, Error: %v\n", err)
  70         }
  71
  72        clientSet, err := kubernetes.NewForConfig(config)
  73         if err != nil {
  74            fmt.Printf("Could not create clientset, Error: %v\n", err)
  75         }
  76        bmhDynamicClient, err := dynamic.NewForConfig(config)
  77
  78        if err != nil {
  79           fmt.Printf("Could not create dynamic client for bareMetalHosts, Error: %v\n", err)
  80        }
  81
  82        return &ReconcileProvisioning{client: mgr.GetClient(), scheme: mgr.GetScheme(), clientset: clientSet, bmhClient: bmhDynamicClient }
  83 }
  84
  85 // add adds a new Controller to mgr with r as the reconcile.Reconciler
  86 func add(mgr manager.Manager, r reconcile.Reconciler) error {
  87         // Create a new controller
  88         c, err := controller.New("provisioning-controller", mgr, controller.Options{Reconciler: r})
  89         if err != nil {
  90                 return err
  91         }
  92
  93         // Watch for changes to primary resource Provisioning
  94         err = c.Watch(&source.Kind{Type: &bpav1alpha1.Provisioning{}}, &handler.EnqueueRequestForObject{})
  95         if err != nil {
  96                 return err
  97         }
  98
  99         // Watch for changes to resource configmap created as a consequence of the provisioning CR
 100         err = c.Watch(&source.Kind{Type: &corev1.ConfigMap{}}, &handler.EnqueueRequestForOwner{
 101                 IsController: true,
 102                 OwnerType:   &bpav1alpha1.Provisioning{},
 103         })
 104
 105         if err != nil {
 106                 return err
 107         }
 108
 109        //Watch for changes to job resource also created as a consequence of the provisioning CR
 110        err = c.Watch(&source.Kind{Type: &batchv1.Job{}}, &handler.EnqueueRequestForOwner{
 111                 IsController: true,
 112                 OwnerType:   &bpav1alpha1.Provisioning{},
 113         })
 114
 115         if err != nil {
 116                 return err
 117         }
 118
 119         // Watch for changes to resource software CR
 120         err = c.Watch(&source.Kind{Type: &bpav1alpha1.Software{}}, &handler.EnqueueRequestForObject{})
 121         if err != nil {
 122                 return err
 123         }
 124
 125
 126         return nil
 127 }
 128
 129 // blank assignment to verify that ReconcileProvisioning implements reconcile.Reconciler
 130 var _ reconcile.Reconciler = &ReconcileProvisioning{}
 131
 132 // ReconcileProvisioning reconciles a Provisioning object
 133 type ReconcileProvisioning struct {
 134         // This client, initialized using mgr.Client() above, is a split client
 135         // that reads objects from the cache and writes to the apiserver
 136         client client.Client
 137         scheme *runtime.Scheme
 138         clientset kubernetes.Interface
 139         bmhClient dynamic.Interface
 140 }
 141
 142 // Reconcile reads that state of the cluster for a Provisioning object and makes changes based on the state read
 143 // and what is in the Provisioning.Spec
 144 // TODO(user): Modify this Reconcile function to implement your Controller logic.  This example creates
 145 // a Pod as an example
 146 // Note:
 147 // The Controller will requeue the Request to be processed again if the returned error is non-nil or
 148 // Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
 149 func (r *ReconcileProvisioning) Reconcile(request reconcile.Request) (reconcile.Result, error) {
 150         reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name)
 151         fmt.Printf("\n\n")
 152         reqLogger.Info("Reconciling Custom Resource")
 153
 154
 155
 156         // Fetch the Provisioning instance
 157         provisioningInstance := &bpav1alpha1.Provisioning{}
 158         softwareInstance := &bpav1alpha1.Software{}
 159         err := r.client.Get(context.TODO(), request.NamespacedName, provisioningInstance)
 160         provisioningCreated := true
 161         if err != nil {
 162
 163                          //Check if its a Software Instance
 164                          err = r.client.Get(context.TODO(), request.NamespacedName, softwareInstance)
 165                          if err != nil {
 166                              if errors.IsNotFound(err) {
 167                                 // Request object not found, could have been deleted after reconcile request.
 168                                 // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers.
 169                                 // Return and don't requeue
 170                                 return reconcile.Result{}, nil
 171                              }
 172
 173                          // Error reading the object - requeue the request.
 174                          return reconcile.Result{}, err
 175                          }
 176
 177                          //No error occured and so a Software CR was created not a Provisoning CR
 178                          provisioningCreated = false
 179         }
 180
 181
 182         masterTag := "MASTER_"
 183         workerTag := "WORKER_"
 184
 185         if provisioningCreated {
 186
 187         ///////////////////////////////////////////////////////////////////////////////////////////////
 188         ////////////////         Provisioning CR was created so install KUD          /////////////////
 189         //////////////////////////////////////////////////////////////////////////////////////////////
 190         clusterName := provisioningInstance.Labels["cluster"]
 191         clusterType := provisioningInstance.Labels["cluster-type"]
 192         mastersList := provisioningInstance.Spec.Masters
 193         workersList := provisioningInstance.Spec.Workers
 194         kudPlugins := provisioningInstance.Spec.KUDPlugins
 195         podSubnet := provisioningInstance.Spec.PodSubnet
 196
 197         bareMetalHostList, _ := listBareMetalHosts(r.bmhClient)
 198         virtletVMList, _ := listVirtletVMs(r.clientset)
 199
 200
 201
 202
 203         var allString string
 204         var masterString string
 205         var workerString string
 206
 207         dhcpLeaseFile := "/var/lib/dhcp/dhcpd.leases"
 208         multiClusterDir := "/multi-cluster"
 209
 210         //Create Directory for the specific cluster
 211         clusterDir := multiClusterDir + "/" + clusterName
 212         os.MkdirAll(clusterDir, os.ModePerm)
 213
 214         //Create Maps to be used for cluster ip address to label configmap
 215         clusterLabel := make(map[string]string)
 216         clusterLabel["cluster"] = clusterName
 217         clusterData := make(map[string]string)
 218
 219
 220
 221        //Iterate through mastersList and get all the mac addresses and IP addresses
 222        for _, masterMap := range mastersList {
 223
 224                 for masterLabel, master := range masterMap {
 225                    masterMAC := master.MACaddress
 226                    hostIPaddress := ""
 227
 228                    if masterMAC == "" {
 229                       err = fmt.Errorf("MAC address for masterNode %s not provided\n", masterLabel)
 230                       return reconcile.Result{}, err
 231                    }
 232
 233                    containsMac, bmhCR := checkMACaddress(bareMetalHostList, masterMAC)
 234
 235                    //Check 'cluster-type' label for Virtlet VMs
 236                    if clusterType == "virtlet-vm" {
 237                        //Get VM IP address of master
 238                        hostIPaddress, err = getVMIPaddress(virtletVMList, masterMAC)
 239                        if err != nil || hostIPaddress == "" {
 240                            err = fmt.Errorf("IP address not found for VM with MAC address %s \n", masterMAC)
 241                            return reconcile.Result{}, err
 242                        }
 243                        containsMac = true
 244                    }
 245
 246                    if containsMac{
 247
 248                        if clusterType != "virtlet-vm" {
 249                            fmt.Printf("BareMetalHost CR %s has NIC with MAC Address %s\n", bmhCR, masterMAC)
 250
 251                            //Get IP address of master
 252                            hostIPaddress, err = getHostIPaddress(masterMAC, dhcpLeaseFile )
 253                            if err != nil || hostIPaddress == ""{
 254                                err = fmt.Errorf("IP address not found for host with MAC address %s \n", masterMAC)
 255                                return reconcile.Result{}, err
 256                            }
 257                            allString += masterLabel + "  ansible_ssh_host="  + hostIPaddress + " ansible_ssh_port=22" + "\n"
 258                        }
 259
 260                        if clusterType == "virtlet-vm" {
 261                            allString += masterLabel + "  ansible_ssh_host="  + hostIPaddress + " ansible_ssh_port=22" + " ansible_ssh_user=root" + " ansible_ssh_pass=root" + "\n"
 262                        }
 263                        masterString += masterLabel + "\n"
 264                        clusterData[masterTag + masterLabel] = hostIPaddress
 265
 266                        fmt.Printf("%s : %s \n", hostIPaddress, masterMAC)
 267
 268                        if len(workersList) != 0 {
 269
 270                            //Iterate through workersList and get all the mac addresses
 271                            for _, workerMap := range workersList {
 272
 273                                //Get worker labels from the workermap
 274                                for workerLabel, worker := range workerMap {
 275
 276                                    //Check if workerString already contains worker label
 277                                    containsWorkerLabel := strings.Contains(workerString, workerLabel)
 278                                    workerMAC := worker.MACaddress
 279                                    hostIPaddress = ""
 280
 281                                    //Error occurs if the same label is given to different hosts (assumption,
 282                                    //each MAC address represents a unique host
 283                                    if workerLabel == masterLabel && workerMAC != masterMAC && workerMAC != "" {
 284                                      if containsWorkerLabel {
 285                                             strings.ReplaceAll(workerString, workerLabel, "")
 286                                          }
 287                                       err = fmt.Errorf(`A node with label %s already exists, modify resource and assign a
 288                                       different label to node with MACAddress %s`, workerLabel, workerMAC)
 289                                       return reconcile.Result{}, err
 290
 291                                    //same node performs worker and master roles
 292                                    } else if workerLabel == masterLabel && !containsWorkerLabel {
 293                                         workerString += workerLabel + "\n"
 294
 295                                         //Add host to ip address config map with worker tag
 296                                         hostIPaddress = clusterData[masterTag + masterLabel]
 297                                         clusterData[workerTag + masterLabel] = hostIPaddress
 298
 299                                    //Error occurs if the same node is given different labels
 300                                    } else if workerLabel != masterLabel && workerMAC == masterMAC {
 301                                          if containsWorkerLabel {
 302                                             strings.ReplaceAll(workerString, workerLabel, "")
 303                                          }
 304                                          err = fmt.Errorf(`A node with label %s already exists, modify resource and assign a
 305                                                         different label to node with MACAddress %s`, workerLabel, workerMAC)
 306                                          return reconcile.Result{}, err
 307
 308                                    //worker node is different from any master node and it has not been added to the worker list
 309                                    } else if workerLabel != masterLabel && !containsWorkerLabel {
 310
 311                                         // Error occurs if MAC address not provided for worker node not matching master
 312                                         if workerMAC == "" {
 313                                           err = fmt.Errorf("MAC address for worker %s not provided", workerLabel)
 314                                           return reconcile.Result{}, err
 315                                          }
 316
 317                                         containsMac, bmhCR := checkMACaddress(bareMetalHostList, workerMAC)
 318
 319                                         if clusterType == "virtlet-vm" {
 320                                             //Get VM IP address of master
 321                                             hostIPaddress, err = getVMIPaddress(virtletVMList, workerMAC)
 322                                             if err != nil || hostIPaddress == "" {
 323                                                 err = fmt.Errorf("IP address not found for VM with MAC address %s \n", workerMAC)
 324                                                 return reconcile.Result{}, err
 325                                             }
 326                                             containsMac = true
 327                                         }
 328
 329                                         if containsMac{
 330
 331                                            if clusterType != "virtlet-vm" {
 332                                                fmt.Printf("Host %s matches that macAddress\n", bmhCR)
 333
 334                                                //Get IP address of worker
 335                                                hostIPaddress, err = getHostIPaddress(workerMAC, dhcpLeaseFile )
 336                                                if err != nil {
 337                                                    fmt.Errorf("IP address not found for host with MAC address %s \n", workerMAC)
 338                                                    return reconcile.Result{}, err
 339                                                }
 340                                                allString += workerLabel + "  ansible_ssh_host="  + hostIPaddress + " ansible_ssh_port=22" + "\n"
 341                                            }
 342                                            fmt.Printf("%s : %s \n", hostIPaddress, workerMAC)
 343
 344                                            if clusterType == "virtlet-vm" {
 345                                                allString += masterLabel + "  ansible_ssh_host="  + hostIPaddress + " ansible_ssh_port=22" + " ansible_ssh_user=root" + " ansible_ssh_pass=root" + "\n"
 346                                            }
 347                                            workerString += workerLabel + "\n"
 348                                            clusterData[workerTag + workerLabel] = hostIPaddress
 349
 350                                        //No host found that matches the worker MAC
 351                                        } else {
 352
 353                                             err = fmt.Errorf("Host with MAC Address %s not found\n", workerMAC)
 354                                             return reconcile.Result{}, err
 355                                           }
 356                                    }
 357                              }
 358                        }
 359                    //No worker node specified, add master as worker node
 360                    } else if len(workersList) == 0 && !strings.Contains(workerString, masterLabel) {
 361                        workerString += masterLabel + "\n"
 362
 363                        //Add host to ip address config map with worker tag
 364                        hostIPaddress = clusterData[masterTag + masterLabel]
 365                        clusterData[workerTag + masterLabel] = hostIPaddress
 366                    }
 367
 368                    //No host matching master MAC found
 369                    } else {
 370                       err = fmt.Errorf("Host with MAC Address %s not found\n", masterMAC)
 371                       return reconcile.Result{}, err
 372                    }
 373              }
 374         }
 375
 376         //Create host.ini file
 377         //iniHostFilePath := kudInstallerScript + "/inventory/hosts.ini"
 378         iniHostFilePath := clusterDir + "/hosts.ini"
 379         newFile, err := os.Create(iniHostFilePath)
 380         defer newFile.Close()
 381
 382
 383         if err != nil {
 384            fmt.Printf("Error occured while creating file \n %v", err)
 385            return reconcile.Result{}, err
 386         }
 387
 388         hostFile, err := ini.Load(iniHostFilePath)
 389         if err != nil {
 390            fmt.Printf("Error occured while Loading file \n %v", err)
 391            return reconcile.Result{}, err
 392         }
 393
 394         _, err = hostFile.NewRawSection("all", allString)
 395         if err != nil {
 396            fmt.Printf("Error occured while creating section \n %v", err)
 397            return reconcile.Result{}, err
 398         }
 399         _, err = hostFile.NewRawSection("kube-master", masterString)
 400         if err != nil {
 401            fmt.Printf("Error occured while creating section \n %v", err)
 402            return reconcile.Result{}, err
 403         }
 404
 405         _, err = hostFile.NewRawSection("kube-node", workerString)
 406         if err != nil {
 407            fmt.Printf("Error occured while creating section \n %v", err)
 408            return reconcile.Result{}, err
 409         }
 410
 411         _, err = hostFile.NewRawSection("etcd", masterString)
 412         if err != nil {
 413            fmt.Printf("Error occured while creating section \n %v", err)
 414            return reconcile.Result{}, err
 415         }
 416
 417         if clusterType != "virtlet-vm" {
 418         _, err = hostFile.NewRawSection("ovn-central", masterString)
 419         if err != nil {
 420            fmt.Printf("Error occured while creating section \n %v", err)
 421            return reconcile.Result{}, err
 422         }
 423
 424         _, err = hostFile.NewRawSection("ovn-controller", workerString)
 425         if err != nil {
 426            fmt.Printf("Error occured while creating section \n %v", err)
 427            return reconcile.Result{}, err
 428         }
 429
 430         _, err = hostFile.NewRawSection("virtlet", workerString)
 431         if err != nil {
 432            fmt.Printf("Error occured while creating section \n %v", err)
 433            return reconcile.Result{}, err
 434         }
 435         }
 436         _, err = hostFile.NewRawSection("k8s-cluster:children", "kube-node\n" + "kube-master")
 437         if err != nil {
 438            fmt.Printf("Error occured while creating section \n %v", err)
 439            return reconcile.Result{}, err
 440         }
 441
 442
 443         //Create host.ini file for KUD
 444         hostFile.SaveTo(iniHostFilePath)
 445
 446         //Install KUD
 447         err = createKUDinstallerJob(clusterName, request.Namespace, clusterLabel,  podSubnet, kudPlugins,  r.clientset)
 448         if err != nil {
 449            fmt.Printf("Error occured while creating KUD Installer job for cluster %v\n ERROR: %v", clusterName, err)
 450            return reconcile.Result{}, err
 451         }
 452
 453         //Start separate thread to keep checking job status, Create an IP address configmap
 454         //for cluster if KUD is successfully installed
 455         go checkJob(clusterName, request.Namespace, clusterData, clusterLabel, r.clientset)
 456
 457         return reconcile.Result{}, nil
 458
 459        }
 460
 461
 462
 463         ///////////////////////////////////////////////////////////////////////////////////////////////
 464         ////////////////         Software CR was created so install software         /////////////////
 465         //////////////////////////////////////////////////////////////////////////////////////////////
 466         softwareClusterName, masterSoftwareList, workerSoftwareList := getSoftwareList(softwareInstance)
 467         defaultSSHPrivateKey := "/root/.ssh/id_rsa"
 468
 469         //Get IP address configmap for the cluster
 470         clusterConfigMapData, err := getConfigMapData(request.Namespace, softwareClusterName, r.clientset)
 471         if err != nil {
 472            fmt.Printf("Error occured while retrieving IP address Data for cluster %s, ERROR: %v\n", softwareClusterName, err)
 473            return reconcile.Result{}, err
 474         }
 475
 476         for hostLabel, ipAddress := range clusterConfigMapData {
 477
 478             if strings.Contains(hostLabel, masterTag) {
 479                // Its a master node, install master software
 480                err = softwareInstaller(ipAddress, defaultSSHPrivateKey, masterSoftwareList)
 481                if err != nil {
 482                   fmt.Printf("Error occured while installing master software in host %s, ERROR: %v\n", hostLabel, err)
 483                }
 484             } else if strings.Contains(hostLabel, workerTag) {
 485               // Its a worker node, install worker software
 486               err = softwareInstaller(ipAddress, defaultSSHPrivateKey, workerSoftwareList)
 487               if err != nil {
 488                   fmt.Printf("Error occured while installing worker software in host %s, ERROR: %v\n", hostLabel, err)
 489                }
 490
 491             }
 492
 493         }
 494
 495         return reconcile.Result{}, nil
 496 }
 497
 498 //Function to Get List containing baremetal hosts
 499 func listBareMetalHosts(bmhDynamicClient dynamic.Interface) (*unstructured.UnstructuredList, error) {
 500
 501     //Create GVR representing a BareMetalHost CR
 502     bmhGVR := schema.GroupVersionResource{
 503       Group:    "metal3.io",
 504       Version:  "v1alpha1",
 505       Resource: "baremetalhosts",
 506     }
 507
 508     //Get List containing all BareMetalHosts CRs
 509     bareMetalHosts, err := bmhDynamicClient.Resource(bmhGVR).List(metav1.ListOptions{})
 510     if err != nil {
 511        fmt.Printf("Error occured, cannot get BareMetalHosts list, Error: %v\n", err)
 512        return &unstructured.UnstructuredList{}, err
 513     }
 514
 515     return bareMetalHosts, nil
 516 }
 517
 518
 519 //Function to check if BareMetalHost containing MAC address exist
 520 func checkMACaddress(bareMetalHostList *unstructured.UnstructuredList, macAddress string) (bool, string) {
 521
 522      //Convert macAddress to byte array for comparison
 523      macAddressByte :=  []byte(macAddress)
 524      macBool := false
 525
 526      for _, bareMetalHost := range bareMetalHostList.Items {
 527          bmhJson, _ := bareMetalHost.MarshalJSON()
 528
 529          macBool = bytes.Contains(bmhJson, macAddressByte)
 530          if macBool{
 531              return macBool, bareMetalHost.GetName()
 532          }
 533
 534       }
 535
 536          return macBool, ""
 537
 538 }
 539
 540
 541 //Function to get the IP address of a host from the DHCP file
 542 func getHostIPaddress(macAddress string, dhcpLeaseFilePath string ) (string, error) {
 543
 544      //Read the dhcp lease file
 545      dhcpFile, err := ioutil.ReadFile(dhcpLeaseFilePath)
 546      if err != nil {
 547         fmt.Printf("Failed to read lease file\n")
 548         return "", err
 549      }
 550
 551      dhcpLeases := string(dhcpFile)
 552
 553      //Regex to use to search dhcpLeases
 554      reg := "lease.*{|ethernet.*|\n. binding state.*"
 555      re, err := regexp.Compile(reg)
 556      if err != nil {
 557         fmt.Printf("Could not create Regexp object, Error %v occured\n", err)
 558         return "", err
 559      }
 560
 561      //Get String containing leased Ip addresses and Corressponding MAC addresses
 562      out := re.FindAllString(dhcpLeases, -1)
 563      outString := strings.Join(out, " ")
 564      stringReplacer := strings.NewReplacer("lease", "", "ethernet ", "", ";", "",
 565      " binding state", "", "{", "")
 566      replaced := stringReplacer.Replace(outString)
 567      ipMacList := strings.Fields(replaced)
 568
 569
 570      //Get IP addresses corresponding to Input MAC Address
 571      for idx := len(ipMacList)-1 ; idx >= 0; idx -- {
 572          item := ipMacList[idx]
 573          if item == macAddress  {
 574
 575             leaseState := ipMacList[idx -1]
 576             if leaseState != "active" {
 577                err := fmt.Errorf("No active ip address lease found for MAC address %s \n", macAddress)
 578                fmt.Printf("%v\n", err)
 579                return "", err
 580             }
 581             ipAdd := ipMacList[idx - 2]
 582             return ipAdd, nil
 583     }
 584
 585  }
 586      return "", nil
 587 }
 588
 589 //Function to create configmap
 590 func createConfigMap(data, labels map[string]string, namespace string, clientset kubernetes.Interface) error{
 591
 592      configmapClient := clientset.CoreV1().ConfigMaps(namespace)
 593
 594      configmap := &corev1.ConfigMap{
 595
 596         ObjectMeta: metav1.ObjectMeta{
 597                         Name: labels["cluster"] + "-configmap",
 598                         Labels: labels,
 599                 },
 600         Data: data,
 601      }
 602
 603
 604       _, err := configmapClient.Create(configmap)
 605       if err != nil {
 606          return err
 607
 608       }
 609       return nil
 610
 611 }
 612
 613 //Function to get configmap Data
 614 func getConfigMapData(namespace, clusterName string, clientset kubernetes.Interface) (map[string]string, error) {
 615
 616      configmapClient := clientset.CoreV1().ConfigMaps(namespace)
 617      configmapName := clusterName + "-configmap"
 618      clusterConfigmap, err := configmapClient.Get(configmapName, metav1.GetOptions{})
 619      if err != nil {
 620         return nil, err
 621      }
 622
 623      configmapData := clusterConfigmap.Data
 624      return configmapData, nil
 625 }
 626
 627 //Function to create job for KUD installation
 628 func createKUDinstallerJob(clusterName, namespace string, labels map[string]string, podSubnet string, kudPlugins []string, clientset kubernetes.Interface) error{
 629
 630     var backOffLimit int32 = 0
 631     var privi bool = true
 632
 633     installerString := " ./installer --cluster " + clusterName + " --network " + podSubnet
 634
 635     // Check if any plugin was specified
 636     if len(kudPlugins) > 0 {
 637             plugins := " --plugins"
 638
 639             for _, plug := range kudPlugins {
 640                plugins += " " + plug
 641             }
 642
 643            installerString += plugins
 644     }
 645
 646
 647     jobClient := clientset.BatchV1().Jobs("default")
 648
 649         job := &batchv1.Job{
 650
 651         ObjectMeta: metav1.ObjectMeta{
 652                         Name: "kud-" + clusterName,
 653                        Labels: labels,
 654                 },
 655                 Spec: batchv1.JobSpec{
 656                       Template: corev1.PodTemplateSpec{
 657                                 ObjectMeta: metav1.ObjectMeta{
 658                                         Labels: labels,
 659                                 },
 660
 661
 662                         Spec: corev1.PodSpec{
 663                               HostNetwork: true,
 664                               Containers: []corev1.Container{{
 665                                           Name: "kud",
 666                                           Image: "github.com/onap/multicloud-k8s:latest",
 667                                           ImagePullPolicy: "IfNotPresent",
 668                                           VolumeMounts: []corev1.VolumeMount{{
 669                                                         Name: "multi-cluster",
 670                                                         MountPath: "/opt/kud/multi-cluster",
 671                                                         },
 672                                                         {
 673                                                         Name: "secret-volume",
 674                                                         MountPath: "/.ssh",
 675                                                         },
 676
 677                                            },
 678                                            Command: []string{"/bin/sh","-c"},
 679                                            Args: []string{"cp -r /.ssh /root/; chmod -R 600 /root/.ssh;" + installerString},
 680                                            SecurityContext: &corev1.SecurityContext{
 681                                                             Privileged : &privi,
 682
 683                                            },
 684                                           },
 685                                  },
 686                                  Volumes: []corev1.Volume{{
 687                                           Name: "multi-cluster",
 688                                           VolumeSource: corev1.VolumeSource{
 689                                                        HostPath: &corev1.HostPathVolumeSource{
 690                                                               Path : "/opt/kud/multi-cluster",
 691                                                      }}},
 692                                           {
 693                                           Name: "secret-volume",
 694                                           VolumeSource: corev1.VolumeSource{
 695                                                         Secret: &corev1.SecretVolumeSource{
 696                                                               SecretName: "ssh-key-secret",
 697                                                         },
 698
 699                                           }}},
 700                                  RestartPolicy: "Never",
 701                               },
 702
 703                              },
 704                              BackoffLimit : &backOffLimit,
 705                              },
 706
 707                          }
 708                     _, err := jobClient.Create(job)
 709                     if err != nil {
 710                        fmt.Printf("ERROR occured while creating job to install KUD\n ERROR:%v", err)
 711                        return err
 712                     }
 713                     return nil
 714
 715 }
 716
 717 //Function to Check if job succeeded
 718 func checkJob(clusterName, namespace string, data, labels map[string]string, clientset kubernetes.Interface) {
 719
 720      fmt.Printf("\nChecking job status for cluster %s\n", clusterName)
 721      jobName := "kud-" + clusterName
 722      jobClient := clientset.BatchV1().Jobs(namespace)
 723
 724      for {
 725          time.Sleep(2 * time.Second)
 726
 727          job, err := jobClient.Get(jobName, metav1.GetOptions{})
 728          if err != nil {
 729             fmt.Printf("ERROR: %v occured while retrieving job: %s", err, jobName)
 730             return
 731          }
 732          jobSucceeded := job.Status.Succeeded
 733          jobFailed := job.Status.Failed
 734
 735          if jobSucceeded == 1 {
 736             fmt.Printf("\n Job succeeded, KUD successfully installed in Cluster %s\n", clusterName)
 737
 738             //KUD was installed successfully create configmap to store IP address info for the cluster
 739             err = createConfigMap(data, labels, namespace, clientset)
 740             if err != nil {
 741                fmt.Printf("Error occured while creating Ip address configmap for cluster %v\n ERROR: %v", clusterName, err)
 742                return
 743             }
 744             return
 745          }
 746
 747         if jobFailed == 1 {
 748            fmt.Printf("\n Job Failed, KUD not installed in Cluster %s, check pod logs\n", clusterName)
 749            return
 750         }
 751
 752      }
 753     return
 754
 755 }
 756
 757 //Function to get software list from software CR
 758 func getSoftwareList(softwareCR *bpav1alpha1.Software) (string, []interface{}, []interface{}) {
 759
 760      CRclusterName := softwareCR.GetLabels()["cluster"]
 761
 762      masterSofwareList := softwareCR.Spec.MasterSoftware
 763      workerSoftwareList := softwareCR.Spec.WorkerSoftware
 764
 765      return CRclusterName, masterSofwareList, workerSoftwareList
 766 }
 767
 768 //Function to install software in clusterHosts
 769 func softwareInstaller(ipAddress, sshPrivateKey string, softwareList []interface{}) error {
 770
 771      var installString string
 772      for _, software := range softwareList {
 773
 774         switch t := software.(type){
 775         case string:
 776              installString += software.(string) + " "
 777         case interface{}:
 778              softwareMap, errBool := software.(map[string]interface{})
 779              if !errBool {
 780                 fmt.Printf("Error occured, cannot install software %v\n", software)
 781              }
 782              for softwareName, versionMap := range softwareMap {
 783
 784                  versionMAP, _ := versionMap.(map[string]interface{})
 785                  version := versionMAP["version"].(string)
 786                  installString += softwareName + "=" + version + " "
 787              }
 788         default:
 789             fmt.Printf("invalid format %v\n", t)
 790         }
 791
 792      }
 793
 794      err := sshInstaller(installString, sshPrivateKey, ipAddress)
 795      if err != nil {
 796         return err
 797      }
 798      return nil
 799
 800 }
 801
 802 //Function to Run Installation commands via ssh
 803 func sshInstaller(softwareString, sshPrivateKey, ipAddress string) error {
 804
 805      buffer, err := ioutil.ReadFile(sshPrivateKey)
 806      if err != nil {
 807         return err
 808      }
 809
 810      key, err := ssh.ParsePrivateKey(buffer)
 811      if err != nil {
 812         return err
 813      }
 814
 815      sshConfig := &ssh.ClientConfig{
 816         User: "root",
 817         Auth: []ssh.AuthMethod{
 818               ssh.PublicKeys(key),
 819      },
 820
 821      HostKeyCallback: ssh.InsecureIgnoreHostKey(),
 822      }
 823
 824     client, err := ssh.Dial("tcp", ipAddress + ":22", sshConfig)
 825     if err != nil {
 826        return err
 827     }
 828
 829     session, err := client.NewSession()
 830     if err != nil {
 831        return err
 832     }
 833
 834     defer session.Close()
 835     defer client.Close()
 836
 837     cmd := "sudo apt-get update && apt-get install " + softwareString + "-y"
 838     err = session.Start(cmd)
 839
 840     if err != nil {
 841        return err
 842     }
 843
 844     return nil
 845
 846 }
 847
 848 func listVirtletVMs(clientset kubernetes.Interface) ([]VirtletVM, error) {
 849
 850         var vmPodList []VirtletVM
 851
 852         pods, err := clientset.CoreV1().Pods("").List(metav1.ListOptions{})
 853         if err != nil {
 854                 fmt.Printf("Could not get pod info, Error: %v\n", err)
 855                 return []VirtletVM{}, err
 856         }
 857
 858         for _, pod := range pods.Items {
 859                 var podAnnotation map[string]interface{}
 860                 var podStatus corev1.PodStatus
 861                 var podDefaultNetStatus []NetworksStatus
 862
 863                 annotation, err := json.Marshal(pod.ObjectMeta.GetAnnotations())
 864                 if err != nil {
 865                         fmt.Printf("Could not get pod annotations, Error: %v\n", err)
 866                         return []VirtletVM{}, err
 867                 }
 868
 869                 json.Unmarshal([]byte(annotation), &podAnnotation)
 870                 if podAnnotation != nil && podAnnotation["kubernetes.io/target-runtime"] != nil {
 871                         runtime := podAnnotation["kubernetes.io/target-runtime"].(string)
 872
 873                         podStatusJson, _ := json.Marshal(pod.Status)
 874                         json.Unmarshal([]byte(podStatusJson), &podStatus)
 875
 876                         if runtime  == "virtlet.cloud" && podStatus.Phase == "Running" && podAnnotation["k8s.v1.cni.cncf.io/networks-status"] != nil {
 877                                 ns := podAnnotation["k8s.v1.cni.cncf.io/networks-status"].(string)
 878                                 json.Unmarshal([]byte(ns), &podDefaultNetStatus)
 879
 880                                 vmPodList = append(vmPodList, VirtletVM{podStatus.PodIP, podDefaultNetStatus[0].Mac})
 881                         }
 882                 }
 883         }
 884
 885         return vmPodList, nil
 886 }
 887
 888 func getVMIPaddress(vmList []VirtletVM, macAddress string) (string, error) {
 889
 890         for i := 0; i < len(vmList); i++ {
 891                 if vmList[i].MACaddress == macAddress {
 892                         return vmList[i].IPaddress, nil
 893                 }
 894         }
 895         return "", nil
 896 }