+
+ //No error occured and so a Software CR was created not a Provisoning CR
+ provisioningCreated = false
+ }
+
+ masterTag := "MASTER_"
+ workerTag := "WORKER_"
+
+ if provisioningCreated {
+
+ ///////////////////////////////////////////////////////////////////////////////////////////////
+ //////////////// Provisioning CR was created so install KUD /////////////////
+ //////////////////////////////////////////////////////////////////////////////////////////////
+ clusterName := provisioningInstance.Labels["cluster"]
+ clusterType := provisioningInstance.Labels["cluster-type"]
+ mastersList := provisioningInstance.Spec.Masters
+ workersList := provisioningInstance.Spec.Workers
+ kudPlugins := provisioningInstance.Spec.KUDPlugins
+ podSubnet := provisioningInstance.Spec.PodSubnet
+
+ bareMetalHostList, _ := listBareMetalHosts(r.bmhClient)
+ virtletVMList, _ := listVirtletVMs(r.clientset)
+
+ var allString string
+ var masterString string
+ var workerString string
+
+ dhcpLeaseFile := "/var/lib/dhcp/dhcpd.leases"
+ multiClusterDir := "/multi-cluster"
+
+ //Create Directory for the specific cluster
+ clusterDir := multiClusterDir + "/" + clusterName
+ os.MkdirAll(clusterDir, os.ModePerm)
+
+ //Create Maps to be used for cluster ip address to label configmap
+ clusterLabel := make(map[string]string)
+ clusterLabel["cluster"] = clusterName
+ clusterData := make(map[string]string)
+
+ //Iterate through mastersList and get all the mac addresses and IP addresses
+ for _, masterMap := range mastersList {
+
+ for masterLabel, master := range masterMap {
+ masterMAC := master.MACaddress
+ hostIPaddress := ""
+
+ if masterMAC == "" {
+ err = fmt.Errorf("MAC address for masterNode %s not provided\n", masterLabel)
+ return reconcile.Result{}, err
+ }
+
+ containsMac, bmhCR := checkMACaddress(bareMetalHostList, masterMAC)
+
+ //Check 'cluster-type' label for Virtlet VMs
+ if clusterType == "virtlet-vm" {
+ //Get VM IP address of master
+ hostIPaddress, err = getVMIPaddress(virtletVMList, masterMAC)
+ if err != nil || hostIPaddress == "" {
+ err = fmt.Errorf("IP address not found for VM with MAC address %s \n", masterMAC)
+ return reconcile.Result{}, err
+ }
+ containsMac = true
+ }
+
+ if containsMac {
+
+ if clusterType != "virtlet-vm" {
+ fmt.Printf("BareMetalHost CR %s has NIC with MAC Address %s\n", bmhCR, masterMAC)
+
+ //Get IP address of master
+ hostIPaddress, err = getHostIPaddress(masterMAC, dhcpLeaseFile)
+ if err != nil || hostIPaddress == "" {
+ err = fmt.Errorf("IP address not found for host with MAC address %s \n", masterMAC)
+ return reconcile.Result{}, err
+ }
+ allString += masterLabel + " ansible_ssh_host=" + hostIPaddress + " ansible_ssh_port=22" + "\n"
+ }
+
+ if clusterType == "virtlet-vm" {
+ allString += masterLabel + " ansible_ssh_host=" + hostIPaddress + " ansible_ssh_port=22" + " ansible_ssh_user=root" + " ansible_ssh_pass=root" + "\n"
+ }
+ masterString += masterLabel + "\n"
+ clusterData[masterTag+masterLabel] = hostIPaddress
+
+ fmt.Printf("%s : %s \n", hostIPaddress, masterMAC)
+
+ if len(workersList) != 0 {
+
+ //Iterate through workersList and get all the mac addresses
+ for _, workerMap := range workersList {
+
+ //Get worker labels from the workermap
+ for workerLabel, worker := range workerMap {
+
+ //Check if workerString already contains worker label
+ containsWorkerLabel := strings.Contains(workerString, workerLabel)
+ workerMAC := worker.MACaddress
+ hostIPaddress = ""
+
+ //Error occurs if the same label is given to different hosts (assumption,
+ //each MAC address represents a unique host
+ if workerLabel == masterLabel && workerMAC != masterMAC && workerMAC != "" {
+ if containsWorkerLabel {
+ strings.ReplaceAll(workerString, workerLabel, "")
+ }
+ err = fmt.Errorf(`A node with label %s already exists, modify resource and assign a
+ different label to node with MACAddress %s`, workerLabel, workerMAC)
+ return reconcile.Result{}, err
+
+ //same node performs worker and master roles
+ } else if workerLabel == masterLabel && !containsWorkerLabel {
+ workerString += workerLabel + "\n"
+
+ //Add host to ip address config map with worker tag
+ hostIPaddress = clusterData[masterTag+masterLabel]
+ clusterData[workerTag+masterLabel] = hostIPaddress
+
+ //Error occurs if the same node is given different labels
+ } else if workerLabel != masterLabel && workerMAC == masterMAC {
+ if containsWorkerLabel {
+ strings.ReplaceAll(workerString, workerLabel, "")
+ }
+ err = fmt.Errorf(`A node with label %s already exists, modify resource and assign a
+ different label to node with MACAddress %s`, workerLabel, workerMAC)
+ return reconcile.Result{}, err
+
+ //worker node is different from any master node and it has not been added to the worker list
+ } else if workerLabel != masterLabel && !containsWorkerLabel {
+
+ // Error occurs if MAC address not provided for worker node not matching master
+ if workerMAC == "" {
+ err = fmt.Errorf("MAC address for worker %s not provided", workerLabel)
+ return reconcile.Result{}, err
+ }
+
+ containsMac, bmhCR := checkMACaddress(bareMetalHostList, workerMAC)
+
+ if clusterType == "virtlet-vm" {
+ //Get VM IP address of master
+ hostIPaddress, err = getVMIPaddress(virtletVMList, workerMAC)
+ if err != nil || hostIPaddress == "" {
+ err = fmt.Errorf("IP address not found for VM with MAC address %s \n", workerMAC)
+ return reconcile.Result{}, err
+ }
+ containsMac = true
+ }
+
+ if containsMac {
+
+ if clusterType != "virtlet-vm" {
+ fmt.Printf("Host %s matches that macAddress\n", bmhCR)
+
+ //Get IP address of worker
+ hostIPaddress, err = getHostIPaddress(workerMAC, dhcpLeaseFile)
+ if err != nil {
+ fmt.Errorf("IP address not found for host with MAC address %s \n", workerMAC)
+ return reconcile.Result{}, err
+ }
+ allString += workerLabel + " ansible_ssh_host=" + hostIPaddress + " ansible_ssh_port=22" + "\n"
+ }
+ fmt.Printf("%s : %s \n", hostIPaddress, workerMAC)
+
+ if clusterType == "virtlet-vm" {
+ allString += masterLabel + " ansible_ssh_host=" + hostIPaddress + " ansible_ssh_port=22" + " ansible_ssh_user=root" + " ansible_ssh_pass=root" + "\n"
+ }
+ workerString += workerLabel + "\n"
+ clusterData[workerTag+workerLabel] = hostIPaddress
+
+ //No host found that matches the worker MAC
+ } else {
+
+ err = fmt.Errorf("Host with MAC Address %s not found\n", workerMAC)
+ return reconcile.Result{}, err
+ }
+ }
+ }
+ }
+ //No worker node specified, add master as worker node
+ } else if len(workersList) == 0 && !strings.Contains(workerString, masterLabel) {
+ workerString += masterLabel + "\n"
+
+ //Add host to ip address config map with worker tag
+ hostIPaddress = clusterData[masterTag+masterLabel]
+ clusterData[workerTag+masterLabel] = hostIPaddress
+ }
+
+ //No host matching master MAC found
+ } else {
+ err = fmt.Errorf("Host with MAC Address %s not found\n", masterMAC)
+ return reconcile.Result{}, err
+ }
+ }
+ }
+
+ //Create host.ini file
+ //iniHostFilePath := kudInstallerScript + "/inventory/hosts.ini"
+ iniHostFilePath := clusterDir + "/hosts.ini"
+ newFile, err := os.Create(iniHostFilePath)
+ defer newFile.Close()
+
+ if err != nil {
+ fmt.Printf("Error occured while creating file \n %v", err)
+ return reconcile.Result{}, err
+ }
+
+ hostFile, err := ini.Load(iniHostFilePath)
+ if err != nil {
+ fmt.Printf("Error occured while Loading file \n %v", err)
+ return reconcile.Result{}, err
+ }
+
+ _, err = hostFile.NewRawSection("all", allString)
+ if err != nil {
+ fmt.Printf("Error occured while creating section \n %v", err)
+ return reconcile.Result{}, err
+ }
+ _, err = hostFile.NewRawSection("kube-master", masterString)
+ if err != nil {
+ fmt.Printf("Error occured while creating section \n %v", err)
+ return reconcile.Result{}, err
+ }
+
+ _, err = hostFile.NewRawSection("kube-node", workerString)
+ if err != nil {
+ fmt.Printf("Error occured while creating section \n %v", err)
+ return reconcile.Result{}, err
+ }
+
+ _, err = hostFile.NewRawSection("etcd", masterString)
+ if err != nil {
+ fmt.Printf("Error occured while creating section \n %v", err)
+ return reconcile.Result{}, err
+ }
+
+ if clusterType != "virtlet-vm" {
+ _, err = hostFile.NewRawSection("ovn-central", masterString)
+ if err != nil {
+ fmt.Printf("Error occured while creating section \n %v", err)
+ return reconcile.Result{}, err
+ }
+
+ _, err = hostFile.NewRawSection("ovn-controller", workerString)
+ if err != nil {
+ fmt.Printf("Error occured while creating section \n %v", err)
+ return reconcile.Result{}, err
+ }
+
+ _, err = hostFile.NewRawSection("virtlet", workerString)
+ if err != nil {
+ fmt.Printf("Error occured while creating section \n %v", err)
+ return reconcile.Result{}, err
+ }
+ }
+ _, err = hostFile.NewRawSection("k8s-cluster:children", "kube-node\n"+"kube-master")
+ if err != nil {
+ fmt.Printf("Error occured while creating section \n %v", err)
+ return reconcile.Result{}, err
+ }
+
+ //Create host.ini file for KUD
+ hostFile.SaveTo(iniHostFilePath)
+
+ //Install KUD
+ err = createKUDinstallerJob(clusterName, request.Namespace, clusterLabel, podSubnet, kudPlugins, r.clientset)
+ if err != nil {
+ fmt.Printf("Error occured while creating KUD Installer job for cluster %v\n ERROR: %v", clusterName, err)
+ return reconcile.Result{}, err
+ }
+
+ //Start separate thread to keep checking job status, Create an IP address configmap
+ //for cluster if KUD is successfully installed
+ go checkJob(clusterName, request.Namespace, clusterData, clusterLabel, r.clientset)
+
+ return reconcile.Result{}, nil
+
+ }
+
+ ///////////////////////////////////////////////////////////////////////////////////////////////
+ //////////////// Software CR was created so install software /////////////////
+ //////////////////////////////////////////////////////////////////////////////////////////////
+ softwareClusterName, masterSoftwareList, workerSoftwareList := getSoftwareList(softwareInstance)
+ defaultSSHPrivateKey := "/root/.ssh/id_rsa"
+
+ //Get IP address configmap for the cluster
+ clusterConfigMapData, err := getConfigMapData(request.Namespace, softwareClusterName, r.clientset)
+ if err != nil {
+ fmt.Printf("Error occured while retrieving IP address Data for cluster %s, ERROR: %v\n", softwareClusterName, err)