From: Todd Malsbary Date: Mon, 14 Jun 2021 20:15:51 +0000 (-0700) Subject: Run go fmt over bpa-operator X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=b98b74555ff21645693f4b0ef12ae36e228bd2f9;p=icn.git Run go fmt over bpa-operator Signed-off-by: Todd Malsbary Change-Id: I7652333843f5099ebc25d3efe70f742243df72ae --- diff --git a/cmd/bpa-operator/pkg/apis/bpa/v1alpha1/provisioning_types.go b/cmd/bpa-operator/pkg/apis/bpa/v1alpha1/provisioning_types.go index d3eb1b4..da92dd2 100644 --- a/cmd/bpa-operator/pkg/apis/bpa/v1alpha1/provisioning_types.go +++ b/cmd/bpa-operator/pkg/apis/bpa/v1alpha1/provisioning_types.go @@ -13,10 +13,10 @@ type ProvisioningSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file // Add custom validation using kubebuilder tags: https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html - Masters []map[string]Master `json:"masters,omitempty"` - Workers []map[string]Worker `json:"workers,omitempty"` - KUDPlugins []string `json:"KUDPlugins,omitempty"` - PodSubnet string `json:"PodSubnet,omitempty"` + Masters []map[string]Master `json:"masters,omitempty"` + Workers []map[string]Worker `json:"workers,omitempty"` + KUDPlugins []string `json:"KUDPlugins,omitempty"` + PodSubnet string `json:"PodSubnet,omitempty"` } // ProvisioningStatus defines the observed state of Provisioning @@ -52,17 +52,17 @@ type ProvisioningList struct { // master struct contains resource requirements for a master node type Master struct { MACaddress string `json:"mac-address,omitempty"` - CPU int32 `json:"cpu,omitempty"` - Memory string `json:"memory,omitempty"` + CPU int32 `json:"cpu,omitempty"` + Memory string `json:"memory,omitempty"` } // worker struct contains resource requirements for a worker node type Worker struct { MACaddress string `json:"mac-address,omitempty"` - CPU int32 `json:"cpu,omitempty"` - Memory string `json:"memory,omitempty"` - SRIOV bool `json:"sriov,omitempty"` - QAT bool `json:"qat,omitempty"` + CPU int32 `json:"cpu,omitempty"` + Memory string `json:"memory,omitempty"` + SRIOV bool `json:"sriov,omitempty"` + QAT bool `json:"qat,omitempty"` } func init() { diff --git a/cmd/bpa-operator/pkg/controller/provisioning/provisioning_controller.go b/cmd/bpa-operator/pkg/controller/provisioning/provisioning_controller.go index da0c18d..6fbe40a 100644 --- a/cmd/bpa-operator/pkg/controller/provisioning/provisioning_controller.go +++ b/cmd/bpa-operator/pkg/controller/provisioning/provisioning_controller.go @@ -1,51 +1,51 @@ package provisioning import ( - "context" - "os" - "fmt" - "time" - "bytes" - "regexp" - "strings" - "io/ioutil" - "encoding/json" - - bpav1alpha1 "github.com/bpa-operator/pkg/apis/bpa/v1alpha1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - corev1 "k8s.io/api/core/v1" - batchv1 "k8s.io/api/batch/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/dynamic" - - "k8s.io/client-go/kubernetes" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/config" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" - "sigs.k8s.io/controller-runtime/pkg/source" - "gopkg.in/ini.v1" + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "regexp" + "strings" + "time" + + bpav1alpha1 "github.com/bpa-operator/pkg/apis/bpa/v1alpha1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "golang.org/x/crypto/ssh" + "gopkg.in/ini.v1" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/source" ) type VirtletVM struct { - IPaddress string - MACaddress string + IPaddress string + MACaddress string } type NetworksStatus struct { - Name string `json:"name,omitempty"` - Interface string `json:"interface,omitempty"` - Ips []string `json:"ips,omitempty"` - Mac string `json:"mac,omitempty"` - Default bool `json:"default,omitempty"` - Dns interface{} `json:"dns,omitempty"` + Name string `json:"name,omitempty"` + Interface string `json:"interface,omitempty"` + Ips []string `json:"ips,omitempty"` + Mac string `json:"mac,omitempty"` + Default bool `json:"default,omitempty"` + Dns interface{} `json:"dns,omitempty"` } var log = logf.Log.WithName("controller_provisioning") @@ -58,72 +58,71 @@ var log = logf.Log.WithName("controller_provisioning") // Add creates a new Provisioning Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(mgr manager.Manager) error { - return add(mgr, newReconciler(mgr)) + return add(mgr, newReconciler(mgr)) } // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) reconcile.Reconciler { - config, err := config.GetConfig() - if err != nil { - fmt.Printf("Could not get kube config, Error: %v\n", err) - } + config, err := config.GetConfig() + if err != nil { + fmt.Printf("Could not get kube config, Error: %v\n", err) + } - clientSet, err := kubernetes.NewForConfig(config) - if err != nil { - fmt.Printf("Could not create clientset, Error: %v\n", err) - } - bmhDynamicClient, err := dynamic.NewForConfig(config) + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + fmt.Printf("Could not create clientset, Error: %v\n", err) + } + bmhDynamicClient, err := dynamic.NewForConfig(config) - if err != nil { - fmt.Printf("Could not create dynamic client for bareMetalHosts, Error: %v\n", err) - } + if err != nil { + fmt.Printf("Could not create dynamic client for bareMetalHosts, Error: %v\n", err) + } - return &ReconcileProvisioning{client: mgr.GetClient(), scheme: mgr.GetScheme(), clientset: clientSet, bmhClient: bmhDynamicClient } + return &ReconcileProvisioning{client: mgr.GetClient(), scheme: mgr.GetScheme(), clientset: clientSet, bmhClient: bmhDynamicClient} } // add adds a new Controller to mgr with r as the reconcile.Reconciler func add(mgr manager.Manager, r reconcile.Reconciler) error { - // Create a new controller - c, err := controller.New("provisioning-controller", mgr, controller.Options{Reconciler: r}) - if err != nil { - return err - } - - // Watch for changes to primary resource Provisioning - err = c.Watch(&source.Kind{Type: &bpav1alpha1.Provisioning{}}, &handler.EnqueueRequestForObject{}) - if err != nil { - return err - } + // Create a new controller + c, err := controller.New("provisioning-controller", mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + + // Watch for changes to primary resource Provisioning + err = c.Watch(&source.Kind{Type: &bpav1alpha1.Provisioning{}}, &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } // Watch for changes to resource configmap created as a consequence of the provisioning CR err = c.Watch(&source.Kind{Type: &corev1.ConfigMap{}}, &handler.EnqueueRequestForOwner{ - IsController: true, - OwnerType: &bpav1alpha1.Provisioning{}, - }) - - if err != nil { - return err - } - - //Watch for changes to job resource also created as a consequence of the provisioning CR - err = c.Watch(&source.Kind{Type: &batchv1.Job{}}, &handler.EnqueueRequestForOwner{ - IsController: true, - OwnerType: &bpav1alpha1.Provisioning{}, - }) - - if err != nil { - return err - } - - // Watch for changes to resource software CR - err = c.Watch(&source.Kind{Type: &bpav1alpha1.Software{}}, &handler.EnqueueRequestForObject{}) - if err != nil { - return err - } - - - return nil + IsController: true, + OwnerType: &bpav1alpha1.Provisioning{}, + }) + + if err != nil { + return err + } + + //Watch for changes to job resource also created as a consequence of the provisioning CR + err = c.Watch(&source.Kind{Type: &batchv1.Job{}}, &handler.EnqueueRequestForOwner{ + IsController: true, + OwnerType: &bpav1alpha1.Provisioning{}, + }) + + if err != nil { + return err + } + + // Watch for changes to resource software CR + err = c.Watch(&source.Kind{Type: &bpav1alpha1.Software{}}, &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } + + return nil } // blank assignment to verify that ReconcileProvisioning implements reconcile.Reconciler @@ -131,12 +130,12 @@ var _ reconcile.Reconciler = &ReconcileProvisioning{} // ReconcileProvisioning reconciles a Provisioning object type ReconcileProvisioning struct { - // This client, initialized using mgr.Client() above, is a split client - // that reads objects from the cache and writes to the apiserver - client client.Client - scheme *runtime.Scheme - clientset kubernetes.Interface - bmhClient dynamic.Interface + // This client, initialized using mgr.Client() above, is a split client + // that reads objects from the cache and writes to the apiserver + client client.Client + scheme *runtime.Scheme + clientset kubernetes.Interface + bmhClient dynamic.Interface } // Reconcile reads that state of the cluster for a Provisioning object and makes changes based on the state read @@ -147,758 +146,735 @@ type ReconcileProvisioning struct { // The Controller will requeue the Request to be processed again if the returned error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileProvisioning) Reconcile(request reconcile.Request) (reconcile.Result, error) { - reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) - fmt.Printf("\n\n") - reqLogger.Info("Reconciling Custom Resource") - - - - // Fetch the Provisioning instance - provisioningInstance := &bpav1alpha1.Provisioning{} - softwareInstance := &bpav1alpha1.Software{} - err := r.client.Get(context.TODO(), request.NamespacedName, provisioningInstance) - provisioningCreated := true - if err != nil { - - //Check if its a Software Instance - err = r.client.Get(context.TODO(), request.NamespacedName, softwareInstance) - if err != nil { - if errors.IsNotFound(err) { - // Request object not found, could have been deleted after reconcile request. - // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. - // Return and don't requeue - return reconcile.Result{}, nil - } - - // Error reading the object - requeue the request. - return reconcile.Result{}, err - } - - //No error occured and so a Software CR was created not a Provisoning CR - provisioningCreated = false - } - - - masterTag := "MASTER_" - workerTag := "WORKER_" - - if provisioningCreated { - - /////////////////////////////////////////////////////////////////////////////////////////////// - //////////////// Provisioning CR was created so install KUD ///////////////// - ////////////////////////////////////////////////////////////////////////////////////////////// - clusterName := provisioningInstance.Labels["cluster"] - clusterType := provisioningInstance.Labels["cluster-type"] - mastersList := provisioningInstance.Spec.Masters - workersList := provisioningInstance.Spec.Workers - kudPlugins := provisioningInstance.Spec.KUDPlugins - podSubnet := provisioningInstance.Spec.PodSubnet - - bareMetalHostList, _ := listBareMetalHosts(r.bmhClient) - virtletVMList, _ := listVirtletVMs(r.clientset) - - - - - var allString string - var masterString string - var workerString string + reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) + fmt.Printf("\n\n") + reqLogger.Info("Reconciling Custom Resource") + + // Fetch the Provisioning instance + provisioningInstance := &bpav1alpha1.Provisioning{} + softwareInstance := &bpav1alpha1.Software{} + err := r.client.Get(context.TODO(), request.NamespacedName, provisioningInstance) + provisioningCreated := true + if err != nil { + + //Check if its a Software Instance + err = r.client.Get(context.TODO(), request.NamespacedName, softwareInstance) + if err != nil { + if errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. + // Return and don't requeue + return reconcile.Result{}, nil + } + + // Error reading the object - requeue the request. + return reconcile.Result{}, err + } + + //No error occured and so a Software CR was created not a Provisoning CR + provisioningCreated = false + } + + masterTag := "MASTER_" + workerTag := "WORKER_" + + if provisioningCreated { + + /////////////////////////////////////////////////////////////////////////////////////////////// + //////////////// Provisioning CR was created so install KUD ///////////////// + ////////////////////////////////////////////////////////////////////////////////////////////// + clusterName := provisioningInstance.Labels["cluster"] + clusterType := provisioningInstance.Labels["cluster-type"] + mastersList := provisioningInstance.Spec.Masters + workersList := provisioningInstance.Spec.Workers + kudPlugins := provisioningInstance.Spec.KUDPlugins + podSubnet := provisioningInstance.Spec.PodSubnet + + bareMetalHostList, _ := listBareMetalHosts(r.bmhClient) + virtletVMList, _ := listVirtletVMs(r.clientset) + + var allString string + var masterString string + var workerString string + + dhcpLeaseFile := "/var/lib/dhcp/dhcpd.leases" + multiClusterDir := "/multi-cluster" + + //Create Directory for the specific cluster + clusterDir := multiClusterDir + "/" + clusterName + os.MkdirAll(clusterDir, os.ModePerm) + + //Create Maps to be used for cluster ip address to label configmap + clusterLabel := make(map[string]string) + clusterLabel["cluster"] = clusterName + clusterData := make(map[string]string) + + //Iterate through mastersList and get all the mac addresses and IP addresses + for _, masterMap := range mastersList { + + for masterLabel, master := range masterMap { + masterMAC := master.MACaddress + hostIPaddress := "" + + if masterMAC == "" { + err = fmt.Errorf("MAC address for masterNode %s not provided\n", masterLabel) + return reconcile.Result{}, err + } + + containsMac, bmhCR := checkMACaddress(bareMetalHostList, masterMAC) + + //Check 'cluster-type' label for Virtlet VMs + if clusterType == "virtlet-vm" { + //Get VM IP address of master + hostIPaddress, err = getVMIPaddress(virtletVMList, masterMAC) + if err != nil || hostIPaddress == "" { + err = fmt.Errorf("IP address not found for VM with MAC address %s \n", masterMAC) + return reconcile.Result{}, err + } + containsMac = true + } + + if containsMac { + + if clusterType != "virtlet-vm" { + fmt.Printf("BareMetalHost CR %s has NIC with MAC Address %s\n", bmhCR, masterMAC) + + //Get IP address of master + hostIPaddress, err = getHostIPaddress(masterMAC, dhcpLeaseFile) + if err != nil || hostIPaddress == "" { + err = fmt.Errorf("IP address not found for host with MAC address %s \n", masterMAC) + return reconcile.Result{}, err + } + allString += masterLabel + " ansible_ssh_host=" + hostIPaddress + " ansible_ssh_port=22" + "\n" + } - dhcpLeaseFile := "/var/lib/dhcp/dhcpd.leases" - multiClusterDir := "/multi-cluster" - - //Create Directory for the specific cluster - clusterDir := multiClusterDir + "/" + clusterName - os.MkdirAll(clusterDir, os.ModePerm) - - //Create Maps to be used for cluster ip address to label configmap - clusterLabel := make(map[string]string) - clusterLabel["cluster"] = clusterName - clusterData := make(map[string]string) - - - - //Iterate through mastersList and get all the mac addresses and IP addresses - for _, masterMap := range mastersList { - - for masterLabel, master := range masterMap { - masterMAC := master.MACaddress - hostIPaddress := "" - - if masterMAC == "" { - err = fmt.Errorf("MAC address for masterNode %s not provided\n", masterLabel) - return reconcile.Result{}, err - } - - containsMac, bmhCR := checkMACaddress(bareMetalHostList, masterMAC) - - //Check 'cluster-type' label for Virtlet VMs - if clusterType == "virtlet-vm" { - //Get VM IP address of master - hostIPaddress, err = getVMIPaddress(virtletVMList, masterMAC) - if err != nil || hostIPaddress == "" { - err = fmt.Errorf("IP address not found for VM with MAC address %s \n", masterMAC) - return reconcile.Result{}, err - } - containsMac = true - } - - if containsMac{ - - if clusterType != "virtlet-vm" { - fmt.Printf("BareMetalHost CR %s has NIC with MAC Address %s\n", bmhCR, masterMAC) - - //Get IP address of master - hostIPaddress, err = getHostIPaddress(masterMAC, dhcpLeaseFile ) - if err != nil || hostIPaddress == ""{ - err = fmt.Errorf("IP address not found for host with MAC address %s \n", masterMAC) - return reconcile.Result{}, err - } - allString += masterLabel + " ansible_ssh_host=" + hostIPaddress + " ansible_ssh_port=22" + "\n" - } - - if clusterType == "virtlet-vm" { - allString += masterLabel + " ansible_ssh_host=" + hostIPaddress + " ansible_ssh_port=22" + " ansible_ssh_user=root" + " ansible_ssh_pass=root" + "\n" - } - masterString += masterLabel + "\n" - clusterData[masterTag + masterLabel] = hostIPaddress + if clusterType == "virtlet-vm" { + allString += masterLabel + " ansible_ssh_host=" + hostIPaddress + " ansible_ssh_port=22" + " ansible_ssh_user=root" + " ansible_ssh_pass=root" + "\n" + } + masterString += masterLabel + "\n" + clusterData[masterTag+masterLabel] = hostIPaddress - fmt.Printf("%s : %s \n", hostIPaddress, masterMAC) + fmt.Printf("%s : %s \n", hostIPaddress, masterMAC) - if len(workersList) != 0 { + if len(workersList) != 0 { - //Iterate through workersList and get all the mac addresses - for _, workerMap := range workersList { + //Iterate through workersList and get all the mac addresses + for _, workerMap := range workersList { - //Get worker labels from the workermap - for workerLabel, worker := range workerMap { + //Get worker labels from the workermap + for workerLabel, worker := range workerMap { - //Check if workerString already contains worker label - containsWorkerLabel := strings.Contains(workerString, workerLabel) - workerMAC := worker.MACaddress - hostIPaddress = "" + //Check if workerString already contains worker label + containsWorkerLabel := strings.Contains(workerString, workerLabel) + workerMAC := worker.MACaddress + hostIPaddress = "" - //Error occurs if the same label is given to different hosts (assumption, - //each MAC address represents a unique host - if workerLabel == masterLabel && workerMAC != masterMAC && workerMAC != "" { - if containsWorkerLabel { - strings.ReplaceAll(workerString, workerLabel, "") - } - err = fmt.Errorf(`A node with label %s already exists, modify resource and assign a + //Error occurs if the same label is given to different hosts (assumption, + //each MAC address represents a unique host + if workerLabel == masterLabel && workerMAC != masterMAC && workerMAC != "" { + if containsWorkerLabel { + strings.ReplaceAll(workerString, workerLabel, "") + } + err = fmt.Errorf(`A node with label %s already exists, modify resource and assign a different label to node with MACAddress %s`, workerLabel, workerMAC) - return reconcile.Result{}, err - - //same node performs worker and master roles - } else if workerLabel == masterLabel && !containsWorkerLabel { - workerString += workerLabel + "\n" - - //Add host to ip address config map with worker tag - hostIPaddress = clusterData[masterTag + masterLabel] - clusterData[workerTag + masterLabel] = hostIPaddress - - //Error occurs if the same node is given different labels - } else if workerLabel != masterLabel && workerMAC == masterMAC { - if containsWorkerLabel { - strings.ReplaceAll(workerString, workerLabel, "") - } - err = fmt.Errorf(`A node with label %s already exists, modify resource and assign a + return reconcile.Result{}, err + + //same node performs worker and master roles + } else if workerLabel == masterLabel && !containsWorkerLabel { + workerString += workerLabel + "\n" + + //Add host to ip address config map with worker tag + hostIPaddress = clusterData[masterTag+masterLabel] + clusterData[workerTag+masterLabel] = hostIPaddress + + //Error occurs if the same node is given different labels + } else if workerLabel != masterLabel && workerMAC == masterMAC { + if containsWorkerLabel { + strings.ReplaceAll(workerString, workerLabel, "") + } + err = fmt.Errorf(`A node with label %s already exists, modify resource and assign a different label to node with MACAddress %s`, workerLabel, workerMAC) - return reconcile.Result{}, err - - //worker node is different from any master node and it has not been added to the worker list - } else if workerLabel != masterLabel && !containsWorkerLabel { - - // Error occurs if MAC address not provided for worker node not matching master - if workerMAC == "" { - err = fmt.Errorf("MAC address for worker %s not provided", workerLabel) - return reconcile.Result{}, err - } - - containsMac, bmhCR := checkMACaddress(bareMetalHostList, workerMAC) - - if clusterType == "virtlet-vm" { - //Get VM IP address of master - hostIPaddress, err = getVMIPaddress(virtletVMList, workerMAC) - if err != nil || hostIPaddress == "" { - err = fmt.Errorf("IP address not found for VM with MAC address %s \n", workerMAC) - return reconcile.Result{}, err - } - containsMac = true - } - - if containsMac{ - - if clusterType != "virtlet-vm" { - fmt.Printf("Host %s matches that macAddress\n", bmhCR) - - //Get IP address of worker - hostIPaddress, err = getHostIPaddress(workerMAC, dhcpLeaseFile ) - if err != nil { - fmt.Errorf("IP address not found for host with MAC address %s \n", workerMAC) - return reconcile.Result{}, err - } - allString += workerLabel + " ansible_ssh_host=" + hostIPaddress + " ansible_ssh_port=22" + "\n" - } - fmt.Printf("%s : %s \n", hostIPaddress, workerMAC) - - if clusterType == "virtlet-vm" { - allString += masterLabel + " ansible_ssh_host=" + hostIPaddress + " ansible_ssh_port=22" + " ansible_ssh_user=root" + " ansible_ssh_pass=root" + "\n" - } - workerString += workerLabel + "\n" - clusterData[workerTag + workerLabel] = hostIPaddress - - //No host found that matches the worker MAC - } else { - - err = fmt.Errorf("Host with MAC Address %s not found\n", workerMAC) - return reconcile.Result{}, err - } - } - } - } - //No worker node specified, add master as worker node - } else if len(workersList) == 0 && !strings.Contains(workerString, masterLabel) { - workerString += masterLabel + "\n" - - //Add host to ip address config map with worker tag - hostIPaddress = clusterData[masterTag + masterLabel] - clusterData[workerTag + masterLabel] = hostIPaddress - } - - //No host matching master MAC found - } else { - err = fmt.Errorf("Host with MAC Address %s not found\n", masterMAC) - return reconcile.Result{}, err - } - } - } - - //Create host.ini file - //iniHostFilePath := kudInstallerScript + "/inventory/hosts.ini" - iniHostFilePath := clusterDir + "/hosts.ini" - newFile, err := os.Create(iniHostFilePath) - defer newFile.Close() - - - if err != nil { - fmt.Printf("Error occured while creating file \n %v", err) - return reconcile.Result{}, err - } - - hostFile, err := ini.Load(iniHostFilePath) - if err != nil { - fmt.Printf("Error occured while Loading file \n %v", err) - return reconcile.Result{}, err - } - - _, err = hostFile.NewRawSection("all", allString) - if err != nil { - fmt.Printf("Error occured while creating section \n %v", err) - return reconcile.Result{}, err - } - _, err = hostFile.NewRawSection("kube-master", masterString) - if err != nil { - fmt.Printf("Error occured while creating section \n %v", err) - return reconcile.Result{}, err - } - - _, err = hostFile.NewRawSection("kube-node", workerString) - if err != nil { - fmt.Printf("Error occured while creating section \n %v", err) - return reconcile.Result{}, err - } - - _, err = hostFile.NewRawSection("etcd", masterString) - if err != nil { - fmt.Printf("Error occured while creating section \n %v", err) - return reconcile.Result{}, err - } - - if clusterType != "virtlet-vm" { - _, err = hostFile.NewRawSection("ovn-central", masterString) - if err != nil { - fmt.Printf("Error occured while creating section \n %v", err) - return reconcile.Result{}, err - } - - _, err = hostFile.NewRawSection("ovn-controller", workerString) - if err != nil { - fmt.Printf("Error occured while creating section \n %v", err) - return reconcile.Result{}, err - } - - _, err = hostFile.NewRawSection("virtlet", workerString) - if err != nil { - fmt.Printf("Error occured while creating section \n %v", err) - return reconcile.Result{}, err - } - } - _, err = hostFile.NewRawSection("k8s-cluster:children", "kube-node\n" + "kube-master") - if err != nil { - fmt.Printf("Error occured while creating section \n %v", err) - return reconcile.Result{}, err - } - - - //Create host.ini file for KUD - hostFile.SaveTo(iniHostFilePath) - - //Install KUD - err = createKUDinstallerJob(clusterName, request.Namespace, clusterLabel, podSubnet, kudPlugins, r.clientset) - if err != nil { - fmt.Printf("Error occured while creating KUD Installer job for cluster %v\n ERROR: %v", clusterName, err) - return reconcile.Result{}, err - } - - //Start separate thread to keep checking job status, Create an IP address configmap - //for cluster if KUD is successfully installed - go checkJob(clusterName, request.Namespace, clusterData, clusterLabel, r.clientset) - - return reconcile.Result{}, nil - - } - - - - /////////////////////////////////////////////////////////////////////////////////////////////// - //////////////// Software CR was created so install software ///////////////// - ////////////////////////////////////////////////////////////////////////////////////////////// - softwareClusterName, masterSoftwareList, workerSoftwareList := getSoftwareList(softwareInstance) - defaultSSHPrivateKey := "/root/.ssh/id_rsa" - - //Get IP address configmap for the cluster - clusterConfigMapData, err := getConfigMapData(request.Namespace, softwareClusterName, r.clientset) - if err != nil { - fmt.Printf("Error occured while retrieving IP address Data for cluster %s, ERROR: %v\n", softwareClusterName, err) - return reconcile.Result{}, err - } - - for hostLabel, ipAddress := range clusterConfigMapData { - - if strings.Contains(hostLabel, masterTag) { - // Its a master node, install master software - err = softwareInstaller(ipAddress, defaultSSHPrivateKey, masterSoftwareList) - if err != nil { - fmt.Printf("Error occured while installing master software in host %s, ERROR: %v\n", hostLabel, err) - } - } else if strings.Contains(hostLabel, workerTag) { - // Its a worker node, install worker software - err = softwareInstaller(ipAddress, defaultSSHPrivateKey, workerSoftwareList) - if err != nil { - fmt.Printf("Error occured while installing worker software in host %s, ERROR: %v\n", hostLabel, err) - } - - } - - } - - return reconcile.Result{}, nil + return reconcile.Result{}, err + + //worker node is different from any master node and it has not been added to the worker list + } else if workerLabel != masterLabel && !containsWorkerLabel { + + // Error occurs if MAC address not provided for worker node not matching master + if workerMAC == "" { + err = fmt.Errorf("MAC address for worker %s not provided", workerLabel) + return reconcile.Result{}, err + } + + containsMac, bmhCR := checkMACaddress(bareMetalHostList, workerMAC) + + if clusterType == "virtlet-vm" { + //Get VM IP address of master + hostIPaddress, err = getVMIPaddress(virtletVMList, workerMAC) + if err != nil || hostIPaddress == "" { + err = fmt.Errorf("IP address not found for VM with MAC address %s \n", workerMAC) + return reconcile.Result{}, err + } + containsMac = true + } + + if containsMac { + + if clusterType != "virtlet-vm" { + fmt.Printf("Host %s matches that macAddress\n", bmhCR) + + //Get IP address of worker + hostIPaddress, err = getHostIPaddress(workerMAC, dhcpLeaseFile) + if err != nil { + fmt.Errorf("IP address not found for host with MAC address %s \n", workerMAC) + return reconcile.Result{}, err + } + allString += workerLabel + " ansible_ssh_host=" + hostIPaddress + " ansible_ssh_port=22" + "\n" + } + fmt.Printf("%s : %s \n", hostIPaddress, workerMAC) + + if clusterType == "virtlet-vm" { + allString += masterLabel + " ansible_ssh_host=" + hostIPaddress + " ansible_ssh_port=22" + " ansible_ssh_user=root" + " ansible_ssh_pass=root" + "\n" + } + workerString += workerLabel + "\n" + clusterData[workerTag+workerLabel] = hostIPaddress + + //No host found that matches the worker MAC + } else { + + err = fmt.Errorf("Host with MAC Address %s not found\n", workerMAC) + return reconcile.Result{}, err + } + } + } + } + //No worker node specified, add master as worker node + } else if len(workersList) == 0 && !strings.Contains(workerString, masterLabel) { + workerString += masterLabel + "\n" + + //Add host to ip address config map with worker tag + hostIPaddress = clusterData[masterTag+masterLabel] + clusterData[workerTag+masterLabel] = hostIPaddress + } + + //No host matching master MAC found + } else { + err = fmt.Errorf("Host with MAC Address %s not found\n", masterMAC) + return reconcile.Result{}, err + } + } + } + + //Create host.ini file + //iniHostFilePath := kudInstallerScript + "/inventory/hosts.ini" + iniHostFilePath := clusterDir + "/hosts.ini" + newFile, err := os.Create(iniHostFilePath) + defer newFile.Close() + + if err != nil { + fmt.Printf("Error occured while creating file \n %v", err) + return reconcile.Result{}, err + } + + hostFile, err := ini.Load(iniHostFilePath) + if err != nil { + fmt.Printf("Error occured while Loading file \n %v", err) + return reconcile.Result{}, err + } + + _, err = hostFile.NewRawSection("all", allString) + if err != nil { + fmt.Printf("Error occured while creating section \n %v", err) + return reconcile.Result{}, err + } + _, err = hostFile.NewRawSection("kube-master", masterString) + if err != nil { + fmt.Printf("Error occured while creating section \n %v", err) + return reconcile.Result{}, err + } + + _, err = hostFile.NewRawSection("kube-node", workerString) + if err != nil { + fmt.Printf("Error occured while creating section \n %v", err) + return reconcile.Result{}, err + } + + _, err = hostFile.NewRawSection("etcd", masterString) + if err != nil { + fmt.Printf("Error occured while creating section \n %v", err) + return reconcile.Result{}, err + } + + if clusterType != "virtlet-vm" { + _, err = hostFile.NewRawSection("ovn-central", masterString) + if err != nil { + fmt.Printf("Error occured while creating section \n %v", err) + return reconcile.Result{}, err + } + + _, err = hostFile.NewRawSection("ovn-controller", workerString) + if err != nil { + fmt.Printf("Error occured while creating section \n %v", err) + return reconcile.Result{}, err + } + + _, err = hostFile.NewRawSection("virtlet", workerString) + if err != nil { + fmt.Printf("Error occured while creating section \n %v", err) + return reconcile.Result{}, err + } + } + _, err = hostFile.NewRawSection("k8s-cluster:children", "kube-node\n"+"kube-master") + if err != nil { + fmt.Printf("Error occured while creating section \n %v", err) + return reconcile.Result{}, err + } + + //Create host.ini file for KUD + hostFile.SaveTo(iniHostFilePath) + + //Install KUD + err = createKUDinstallerJob(clusterName, request.Namespace, clusterLabel, podSubnet, kudPlugins, r.clientset) + if err != nil { + fmt.Printf("Error occured while creating KUD Installer job for cluster %v\n ERROR: %v", clusterName, err) + return reconcile.Result{}, err + } + + //Start separate thread to keep checking job status, Create an IP address configmap + //for cluster if KUD is successfully installed + go checkJob(clusterName, request.Namespace, clusterData, clusterLabel, r.clientset) + + return reconcile.Result{}, nil + + } + + /////////////////////////////////////////////////////////////////////////////////////////////// + //////////////// Software CR was created so install software ///////////////// + ////////////////////////////////////////////////////////////////////////////////////////////// + softwareClusterName, masterSoftwareList, workerSoftwareList := getSoftwareList(softwareInstance) + defaultSSHPrivateKey := "/root/.ssh/id_rsa" + + //Get IP address configmap for the cluster + clusterConfigMapData, err := getConfigMapData(request.Namespace, softwareClusterName, r.clientset) + if err != nil { + fmt.Printf("Error occured while retrieving IP address Data for cluster %s, ERROR: %v\n", softwareClusterName, err) + return reconcile.Result{}, err + } + + for hostLabel, ipAddress := range clusterConfigMapData { + + if strings.Contains(hostLabel, masterTag) { + // Its a master node, install master software + err = softwareInstaller(ipAddress, defaultSSHPrivateKey, masterSoftwareList) + if err != nil { + fmt.Printf("Error occured while installing master software in host %s, ERROR: %v\n", hostLabel, err) + } + } else if strings.Contains(hostLabel, workerTag) { + // Its a worker node, install worker software + err = softwareInstaller(ipAddress, defaultSSHPrivateKey, workerSoftwareList) + if err != nil { + fmt.Printf("Error occured while installing worker software in host %s, ERROR: %v\n", hostLabel, err) + } + + } + + } + + return reconcile.Result{}, nil } //Function to Get List containing baremetal hosts func listBareMetalHosts(bmhDynamicClient dynamic.Interface) (*unstructured.UnstructuredList, error) { - //Create GVR representing a BareMetalHost CR - bmhGVR := schema.GroupVersionResource{ - Group: "metal3.io", - Version: "v1alpha1", - Resource: "baremetalhosts", - } - - //Get List containing all BareMetalHosts CRs - bareMetalHosts, err := bmhDynamicClient.Resource(bmhGVR).List(metav1.ListOptions{}) - if err != nil { - fmt.Printf("Error occured, cannot get BareMetalHosts list, Error: %v\n", err) - return &unstructured.UnstructuredList{}, err - } - - return bareMetalHosts, nil + //Create GVR representing a BareMetalHost CR + bmhGVR := schema.GroupVersionResource{ + Group: "metal3.io", + Version: "v1alpha1", + Resource: "baremetalhosts", + } + + //Get List containing all BareMetalHosts CRs + bareMetalHosts, err := bmhDynamicClient.Resource(bmhGVR).List(metav1.ListOptions{}) + if err != nil { + fmt.Printf("Error occured, cannot get BareMetalHosts list, Error: %v\n", err) + return &unstructured.UnstructuredList{}, err + } + + return bareMetalHosts, nil } - //Function to check if BareMetalHost containing MAC address exist func checkMACaddress(bareMetalHostList *unstructured.UnstructuredList, macAddress string) (bool, string) { - //Convert macAddress to byte array for comparison - macAddressByte := []byte(macAddress) - macBool := false + //Convert macAddress to byte array for comparison + macAddressByte := []byte(macAddress) + macBool := false - for _, bareMetalHost := range bareMetalHostList.Items { - bmhJson, _ := bareMetalHost.MarshalJSON() + for _, bareMetalHost := range bareMetalHostList.Items { + bmhJson, _ := bareMetalHost.MarshalJSON() - macBool = bytes.Contains(bmhJson, macAddressByte) - if macBool{ - return macBool, bareMetalHost.GetName() - } + macBool = bytes.Contains(bmhJson, macAddressByte) + if macBool { + return macBool, bareMetalHost.GetName() + } - } + } - return macBool, "" + return macBool, "" } - //Function to get the IP address of a host from the DHCP file -func getHostIPaddress(macAddress string, dhcpLeaseFilePath string ) (string, error) { - - //Read the dhcp lease file - dhcpFile, err := ioutil.ReadFile(dhcpLeaseFilePath) - if err != nil { - fmt.Printf("Failed to read lease file\n") - return "", err - } - - dhcpLeases := string(dhcpFile) - - //Regex to use to search dhcpLeases - reg := "lease.*{|ethernet.*|\n. binding state.*" - re, err := regexp.Compile(reg) - if err != nil { - fmt.Printf("Could not create Regexp object, Error %v occured\n", err) - return "", err - } - - //Get String containing leased Ip addresses and Corressponding MAC addresses - out := re.FindAllString(dhcpLeases, -1) - outString := strings.Join(out, " ") - stringReplacer := strings.NewReplacer("lease", "", "ethernet ", "", ";", "", - " binding state", "", "{", "") - replaced := stringReplacer.Replace(outString) - ipMacList := strings.Fields(replaced) - - - //Get IP addresses corresponding to Input MAC Address - for idx := len(ipMacList)-1 ; idx >= 0; idx -- { - item := ipMacList[idx] - if item == macAddress { - - leaseState := ipMacList[idx -1] - if leaseState != "active" { - err := fmt.Errorf("No active ip address lease found for MAC address %s \n", macAddress) - fmt.Printf("%v\n", err) - return "", err - } - ipAdd := ipMacList[idx - 2] - return ipAdd, nil - } - - } - return "", nil +func getHostIPaddress(macAddress string, dhcpLeaseFilePath string) (string, error) { + + //Read the dhcp lease file + dhcpFile, err := ioutil.ReadFile(dhcpLeaseFilePath) + if err != nil { + fmt.Printf("Failed to read lease file\n") + return "", err + } + + dhcpLeases := string(dhcpFile) + + //Regex to use to search dhcpLeases + reg := "lease.*{|ethernet.*|\n. binding state.*" + re, err := regexp.Compile(reg) + if err != nil { + fmt.Printf("Could not create Regexp object, Error %v occured\n", err) + return "", err + } + + //Get String containing leased Ip addresses and Corressponding MAC addresses + out := re.FindAllString(dhcpLeases, -1) + outString := strings.Join(out, " ") + stringReplacer := strings.NewReplacer("lease", "", "ethernet ", "", ";", "", + " binding state", "", "{", "") + replaced := stringReplacer.Replace(outString) + ipMacList := strings.Fields(replaced) + + //Get IP addresses corresponding to Input MAC Address + for idx := len(ipMacList) - 1; idx >= 0; idx-- { + item := ipMacList[idx] + if item == macAddress { + + leaseState := ipMacList[idx-1] + if leaseState != "active" { + err := fmt.Errorf("No active ip address lease found for MAC address %s \n", macAddress) + fmt.Printf("%v\n", err) + return "", err + } + ipAdd := ipMacList[idx-2] + return ipAdd, nil + } + + } + return "", nil } -//Function to create configmap -func createConfigMap(data, labels map[string]string, namespace string, clientset kubernetes.Interface) error{ - - configmapClient := clientset.CoreV1().ConfigMaps(namespace) +//Function to create configmap +func createConfigMap(data, labels map[string]string, namespace string, clientset kubernetes.Interface) error { - configmap := &corev1.ConfigMap{ + configmapClient := clientset.CoreV1().ConfigMaps(namespace) - ObjectMeta: metav1.ObjectMeta{ - Name: labels["cluster"] + "-configmap", - Labels: labels, - }, - Data: data, - } + configmap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: labels["cluster"] + "-configmap", + Labels: labels, + }, + Data: data, + } - _, err := configmapClient.Create(configmap) - if err != nil { - return err + _, err := configmapClient.Create(configmap) + if err != nil { + return err - } - return nil + } + return nil } //Function to get configmap Data func getConfigMapData(namespace, clusterName string, clientset kubernetes.Interface) (map[string]string, error) { - configmapClient := clientset.CoreV1().ConfigMaps(namespace) - configmapName := clusterName + "-configmap" - clusterConfigmap, err := configmapClient.Get(configmapName, metav1.GetOptions{}) - if err != nil { - return nil, err - } + configmapClient := clientset.CoreV1().ConfigMaps(namespace) + configmapName := clusterName + "-configmap" + clusterConfigmap, err := configmapClient.Get(configmapName, metav1.GetOptions{}) + if err != nil { + return nil, err + } - configmapData := clusterConfigmap.Data - return configmapData, nil + configmapData := clusterConfigmap.Data + return configmapData, nil } //Function to create job for KUD installation -func createKUDinstallerJob(clusterName, namespace string, labels map[string]string, podSubnet string, kudPlugins []string, clientset kubernetes.Interface) error{ - - var backOffLimit int32 = 0 - var privi bool = true - - installerString := " ./installer --cluster " + clusterName - if len(podSubnet) > 0 { - installerString += " --network " + podSubnet - } - - // Check if any plugin was specified - if len(kudPlugins) > 0 { - plugins := " --plugins" - - for _, plug := range kudPlugins { - plugins += " " + plug - } - - installerString += plugins - } - - - jobClient := clientset.BatchV1().Jobs("default") - - job := &batchv1.Job{ - - ObjectMeta: metav1.ObjectMeta{ - Name: "kud-" + clusterName, - Labels: labels, - }, - Spec: batchv1.JobSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - - - Spec: corev1.PodSpec{ - HostNetwork: true, - Containers: []corev1.Container{{ - Name: "kud", - Image: "github.com/onap/multicloud-k8s:latest", - ImagePullPolicy: "IfNotPresent", - VolumeMounts: []corev1.VolumeMount{{ - Name: "multi-cluster", - MountPath: "/opt/kud/multi-cluster", - }, - { - Name: "secret-volume", - MountPath: "/.ssh", - }, - - }, - EnvFrom: []corev1.EnvFromSource{ - { - ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "kud-installer"}}, - }, - }, - Command: []string{"/bin/sh","-c"}, - Args: []string{"cp -r /.ssh /root/; chmod -R 600 /root/.ssh;" + installerString}, - SecurityContext: &corev1.SecurityContext{ - Privileged : &privi, - - }, - }, - }, - Volumes: []corev1.Volume{{ - Name: "multi-cluster", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path : "/opt/kud/multi-cluster", - }}}, - { - Name: "secret-volume", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "ssh-key-secret", - }, - - }}}, - RestartPolicy: "Never", - }, - - }, - BackoffLimit : &backOffLimit, - }, - - } - _, err := jobClient.Create(job) - if err != nil { - fmt.Printf("ERROR occured while creating job to install KUD\n ERROR:%v", err) - return err - } - return nil +func createKUDinstallerJob(clusterName, namespace string, labels map[string]string, podSubnet string, kudPlugins []string, clientset kubernetes.Interface) error { + + var backOffLimit int32 = 0 + var privi bool = true + + installerString := " ./installer --cluster " + clusterName + if len(podSubnet) > 0 { + installerString += " --network " + podSubnet + } + + // Check if any plugin was specified + if len(kudPlugins) > 0 { + plugins := " --plugins" + + for _, plug := range kudPlugins { + plugins += " " + plug + } + + installerString += plugins + } + + jobClient := clientset.BatchV1().Jobs("default") + + job := &batchv1.Job{ + + ObjectMeta: metav1.ObjectMeta{ + Name: "kud-" + clusterName, + Labels: labels, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + + Spec: corev1.PodSpec{ + HostNetwork: true, + Containers: []corev1.Container{{ + Name: "kud", + Image: "github.com/onap/multicloud-k8s:latest", + ImagePullPolicy: "IfNotPresent", + VolumeMounts: []corev1.VolumeMount{{ + Name: "multi-cluster", + MountPath: "/opt/kud/multi-cluster", + }, + { + Name: "secret-volume", + MountPath: "/.ssh", + }, + }, + EnvFrom: []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "kud-installer"}}, + }, + }, + Command: []string{"/bin/sh", "-c"}, + Args: []string{"cp -r /.ssh /root/; chmod -R 600 /root/.ssh;" + installerString}, + SecurityContext: &corev1.SecurityContext{ + Privileged: &privi, + }, + }, + }, + Volumes: []corev1.Volume{{ + Name: "multi-cluster", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/opt/kud/multi-cluster", + }}}, + { + Name: "secret-volume", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "ssh-key-secret", + }, + }}}, + RestartPolicy: "Never", + }, + }, + BackoffLimit: &backOffLimit, + }, + } + _, err := jobClient.Create(job) + if err != nil { + fmt.Printf("ERROR occured while creating job to install KUD\n ERROR:%v", err) + return err + } + return nil } //Function to Check if job succeeded func checkJob(clusterName, namespace string, data, labels map[string]string, clientset kubernetes.Interface) { - fmt.Printf("\nChecking job status for cluster %s\n", clusterName) - jobName := "kud-" + clusterName - jobClient := clientset.BatchV1().Jobs(namespace) - - for { - time.Sleep(2 * time.Second) - - job, err := jobClient.Get(jobName, metav1.GetOptions{}) - if err != nil { - fmt.Printf("ERROR: %v occured while retrieving job: %s", err, jobName) - return - } - jobSucceeded := job.Status.Succeeded - jobFailed := job.Status.Failed - - if jobSucceeded == 1 { - fmt.Printf("\n Job succeeded, KUD successfully installed in Cluster %s\n", clusterName) - - //KUD was installed successfully create configmap to store IP address info for the cluster - err = createConfigMap(data, labels, namespace, clientset) - if err != nil { - fmt.Printf("Error occured while creating Ip address configmap for cluster %v\n ERROR: %v", clusterName, err) - return - } - return - } - - if jobFailed == 1 { - fmt.Printf("\n Job Failed, KUD not installed in Cluster %s, check pod logs\n", clusterName) - return - } - - } - return + fmt.Printf("\nChecking job status for cluster %s\n", clusterName) + jobName := "kud-" + clusterName + jobClient := clientset.BatchV1().Jobs(namespace) + + for { + time.Sleep(2 * time.Second) + + job, err := jobClient.Get(jobName, metav1.GetOptions{}) + if err != nil { + fmt.Printf("ERROR: %v occured while retrieving job: %s", err, jobName) + return + } + jobSucceeded := job.Status.Succeeded + jobFailed := job.Status.Failed + + if jobSucceeded == 1 { + fmt.Printf("\n Job succeeded, KUD successfully installed in Cluster %s\n", clusterName) + + //KUD was installed successfully create configmap to store IP address info for the cluster + err = createConfigMap(data, labels, namespace, clientset) + if err != nil { + fmt.Printf("Error occured while creating Ip address configmap for cluster %v\n ERROR: %v", clusterName, err) + return + } + return + } + + if jobFailed == 1 { + fmt.Printf("\n Job Failed, KUD not installed in Cluster %s, check pod logs\n", clusterName) + return + } + + } + return } //Function to get software list from software CR func getSoftwareList(softwareCR *bpav1alpha1.Software) (string, []interface{}, []interface{}) { - CRclusterName := softwareCR.GetLabels()["cluster"] + CRclusterName := softwareCR.GetLabels()["cluster"] - masterSofwareList := softwareCR.Spec.MasterSoftware - workerSoftwareList := softwareCR.Spec.WorkerSoftware + masterSofwareList := softwareCR.Spec.MasterSoftware + workerSoftwareList := softwareCR.Spec.WorkerSoftware - return CRclusterName, masterSofwareList, workerSoftwareList + return CRclusterName, masterSofwareList, workerSoftwareList } //Function to install software in clusterHosts func softwareInstaller(ipAddress, sshPrivateKey string, softwareList []interface{}) error { - var installString string - for _, software := range softwareList { - - switch t := software.(type){ - case string: - installString += software.(string) + " " - case interface{}: - softwareMap, errBool := software.(map[string]interface{}) - if !errBool { - fmt.Printf("Error occured, cannot install software %v\n", software) - } - for softwareName, versionMap := range softwareMap { - - versionMAP, _ := versionMap.(map[string]interface{}) - version := versionMAP["version"].(string) - installString += softwareName + "=" + version + " " - } - default: - fmt.Printf("invalid format %v\n", t) - } - - } - - err := sshInstaller(installString, sshPrivateKey, ipAddress) - if err != nil { - return err - } - return nil + var installString string + for _, software := range softwareList { + + switch t := software.(type) { + case string: + installString += software.(string) + " " + case interface{}: + softwareMap, errBool := software.(map[string]interface{}) + if !errBool { + fmt.Printf("Error occured, cannot install software %v\n", software) + } + for softwareName, versionMap := range softwareMap { + + versionMAP, _ := versionMap.(map[string]interface{}) + version := versionMAP["version"].(string) + installString += softwareName + "=" + version + " " + } + default: + fmt.Printf("invalid format %v\n", t) + } + + } + + err := sshInstaller(installString, sshPrivateKey, ipAddress) + if err != nil { + return err + } + return nil } //Function to Run Installation commands via ssh func sshInstaller(softwareString, sshPrivateKey, ipAddress string) error { - buffer, err := ioutil.ReadFile(sshPrivateKey) - if err != nil { - return err - } + buffer, err := ioutil.ReadFile(sshPrivateKey) + if err != nil { + return err + } - key, err := ssh.ParsePrivateKey(buffer) - if err != nil { - return err - } + key, err := ssh.ParsePrivateKey(buffer) + if err != nil { + return err + } - sshConfig := &ssh.ClientConfig{ - User: "root", - Auth: []ssh.AuthMethod{ - ssh.PublicKeys(key), - }, + sshConfig := &ssh.ClientConfig{ + User: "root", + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(key), + }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } - client, err := ssh.Dial("tcp", ipAddress + ":22", sshConfig) - if err != nil { - return err - } + client, err := ssh.Dial("tcp", ipAddress+":22", sshConfig) + if err != nil { + return err + } - session, err := client.NewSession() - if err != nil { - return err - } + session, err := client.NewSession() + if err != nil { + return err + } - defer session.Close() - defer client.Close() + defer session.Close() + defer client.Close() - cmd := "sudo apt-get update && apt-get install " + softwareString + "-y" - err = session.Start(cmd) + cmd := "sudo apt-get update && apt-get install " + softwareString + "-y" + err = session.Start(cmd) - if err != nil { - return err - } + if err != nil { + return err + } - return nil + return nil } func listVirtletVMs(clientset kubernetes.Interface) ([]VirtletVM, error) { - var vmPodList []VirtletVM + var vmPodList []VirtletVM - pods, err := clientset.CoreV1().Pods("").List(metav1.ListOptions{}) - if err != nil { - fmt.Printf("Could not get pod info, Error: %v\n", err) - return []VirtletVM{}, err - } + pods, err := clientset.CoreV1().Pods("").List(metav1.ListOptions{}) + if err != nil { + fmt.Printf("Could not get pod info, Error: %v\n", err) + return []VirtletVM{}, err + } - for _, pod := range pods.Items { - var podAnnotation map[string]interface{} - var podStatus corev1.PodStatus - var podDefaultNetStatus []NetworksStatus + for _, pod := range pods.Items { + var podAnnotation map[string]interface{} + var podStatus corev1.PodStatus + var podDefaultNetStatus []NetworksStatus - annotation, err := json.Marshal(pod.ObjectMeta.GetAnnotations()) - if err != nil { - fmt.Printf("Could not get pod annotations, Error: %v\n", err) - return []VirtletVM{}, err - } + annotation, err := json.Marshal(pod.ObjectMeta.GetAnnotations()) + if err != nil { + fmt.Printf("Could not get pod annotations, Error: %v\n", err) + return []VirtletVM{}, err + } - json.Unmarshal([]byte(annotation), &podAnnotation) - if podAnnotation != nil && podAnnotation["kubernetes.io/target-runtime"] != nil { - runtime := podAnnotation["kubernetes.io/target-runtime"].(string) + json.Unmarshal([]byte(annotation), &podAnnotation) + if podAnnotation != nil && podAnnotation["kubernetes.io/target-runtime"] != nil { + runtime := podAnnotation["kubernetes.io/target-runtime"].(string) - podStatusJson, _ := json.Marshal(pod.Status) - json.Unmarshal([]byte(podStatusJson), &podStatus) + podStatusJson, _ := json.Marshal(pod.Status) + json.Unmarshal([]byte(podStatusJson), &podStatus) - if runtime == "virtlet.cloud" && podStatus.Phase == "Running" && podAnnotation["k8s.v1.cni.cncf.io/networks-status"] != nil { - ns := podAnnotation["k8s.v1.cni.cncf.io/networks-status"].(string) - json.Unmarshal([]byte(ns), &podDefaultNetStatus) + if runtime == "virtlet.cloud" && podStatus.Phase == "Running" && podAnnotation["k8s.v1.cni.cncf.io/networks-status"] != nil { + ns := podAnnotation["k8s.v1.cni.cncf.io/networks-status"].(string) + json.Unmarshal([]byte(ns), &podDefaultNetStatus) - vmPodList = append(vmPodList, VirtletVM{podStatus.PodIP, podDefaultNetStatus[0].Mac}) - } - } - } + vmPodList = append(vmPodList, VirtletVM{podStatus.PodIP, podDefaultNetStatus[0].Mac}) + } + } + } - return vmPodList, nil + return vmPodList, nil } func getVMIPaddress(vmList []VirtletVM, macAddress string) (string, error) { - for i := 0; i < len(vmList); i++ { - if vmList[i].MACaddress == macAddress { - return vmList[i].IPaddress, nil - } - } - return "", nil + for i := 0; i < len(vmList); i++ { + if vmList[i].MACaddress == macAddress { + return vmList[i].IPaddress, nil + } + } + return "", nil } diff --git a/cmd/bpa-operator/pkg/controller/provisioning/provisioning_controller_test.go b/cmd/bpa-operator/pkg/controller/provisioning/provisioning_controller_test.go index 9ec0aa5..b305aaa 100644 --- a/cmd/bpa-operator/pkg/controller/provisioning/provisioning_controller_test.go +++ b/cmd/bpa-operator/pkg/controller/provisioning/provisioning_controller_test.go @@ -1,123 +1,120 @@ package provisioning import ( - - "testing" - "io/ioutil" - "os" - - bpav1alpha1 "github.com/bpa-operator/pkg/apis/bpa/v1alpha1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - fakedynamic "k8s.io/client-go/dynamic/fake" - fakeclientset "k8s.io/client-go/kubernetes/fake" + "io/ioutil" + "os" + "testing" + + bpav1alpha1 "github.com/bpa-operator/pkg/apis/bpa/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + fakedynamic "k8s.io/client-go/dynamic/fake" + fakeclientset "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" ) func TestProvisioningController(t *testing.T) { - logf.SetLogger(logf.ZapLogger(true)) - bpaName1 := "bpa-test-cr" - bpaName2 := "bpa-test-2" - bpaName3 := "bpa-test-3" - namespace := "default" - clusterName := "test-cluster" - clusterName2 := "test-cluster-2" - clusterName3 := "test-cluster-3" - macAddress1 := "08:00:27:00:ab:2c" - macAddress2 := "08:00:27:00:ab:3d" - macAddress3 := "08:00:27:00:ab:1c" - - // Create Fake DHCP file - err := createFakeDHCP() - if err != nil { - t.Fatalf("Cannot create Fake DHCP file for testing\n") - } - - // Create Fake baremetalhost - bmhList := newBMList() - - // Create Fake Provisioning CR - provisioning := newBPA(bpaName1, namespace, clusterName, macAddress1) - provisioning2 := newBPA(bpaName2, namespace, clusterName2, macAddress2) - provisioning3 := newBPA(bpaName3, namespace, clusterName3, macAddress3) - - // Objects to track in the fake Client - objs := []runtime.Object{provisioning, provisioning2, provisioning3} - - // Register operator types with the runtime scheme - sc := scheme.Scheme - - sc.AddKnownTypes(bpav1alpha1.SchemeGroupVersion, provisioning, provisioning2, provisioning3) - - // Create Fake Clients and Clientset - fakeClient := fake.NewFakeClient(objs...) - fakeDyn := fakedynamic.NewSimpleDynamicClient(sc, bmhList,) - fakeClientSet := fakeclientset.NewSimpleClientset() - - r := &ReconcileProvisioning{client: fakeClient, scheme: sc, clientset: fakeClientSet, bmhClient: fakeDyn} - - // Mock request to simulate Reconcile() being called on an event for a watched resource - req := simulateRequest(provisioning) - _, err = r.Reconcile(req) - if err != nil { - t.Fatalf("reconcile: (%v)", err) - } - - // Test 1: Check the job was created with the expected name - jobClient := r.clientset.BatchV1().Jobs(namespace) - job, err := jobClient.Get("kud-test-cluster", metav1.GetOptions{}) - - if err != nil { - t.Fatalf("Error occured while getting job: (%v)", err) - } - - // Test 2: Check that cluster name metadata in job is the expected cluster name - jobClusterName := job.Labels["cluster"] - if jobClusterName != clusterName { - t.Fatalf("Job cluster Name is wrong") - } - - - // Test 3: Check that the right error is produced when host with MAC address does not exist - req = simulateRequest(provisioning2) - _, err = r.Reconcile(req) - expectedErr := "Host with MAC Address " + macAddress2 + " not found\n" - if err.Error() != expectedErr { - t.Fatalf("Failed, Unexpected error occured %v\n", err) - } - - // Test 4: Check that the right error is produced when MAC address is not found in the DHCP lease file - req = simulateRequest(provisioning3) - _, err = r.Reconcile(req) - expectedErr = "IP address not found for host with MAC address " + macAddress3 + " \n" - if err.Error() != expectedErr { - t.Fatalf("Failed, Unexpected error occured %v\n", err) - } - - // Delete Fake DHCP file and cluster directories - err = os.Remove("/var/lib/dhcp/dhcpd.leases") - if err != nil { - t.Logf("\nUnable to delete fake DHCP file\n") - } - err = os.RemoveAll("/multi-cluster/" + clusterName) - if err != nil { - t.Logf("\nUnable to delete cluster directory %s\n", clusterName) - } - err = os.RemoveAll("/multi-cluster/" + clusterName2) - if err != nil { - t.Logf("\nUnable to delete cluster directory %s\n", clusterName2) - } - err = os.RemoveAll("/multi-cluster/" + clusterName3) - if err != nil { - t.Logf("\nUnable to delete cluster directory %s\n", clusterName3) - } + logf.SetLogger(logf.ZapLogger(true)) + bpaName1 := "bpa-test-cr" + bpaName2 := "bpa-test-2" + bpaName3 := "bpa-test-3" + namespace := "default" + clusterName := "test-cluster" + clusterName2 := "test-cluster-2" + clusterName3 := "test-cluster-3" + macAddress1 := "08:00:27:00:ab:2c" + macAddress2 := "08:00:27:00:ab:3d" + macAddress3 := "08:00:27:00:ab:1c" + + // Create Fake DHCP file + err := createFakeDHCP() + if err != nil { + t.Fatalf("Cannot create Fake DHCP file for testing\n") + } + + // Create Fake baremetalhost + bmhList := newBMList() + + // Create Fake Provisioning CR + provisioning := newBPA(bpaName1, namespace, clusterName, macAddress1) + provisioning2 := newBPA(bpaName2, namespace, clusterName2, macAddress2) + provisioning3 := newBPA(bpaName3, namespace, clusterName3, macAddress3) + + // Objects to track in the fake Client + objs := []runtime.Object{provisioning, provisioning2, provisioning3} + + // Register operator types with the runtime scheme + sc := scheme.Scheme + + sc.AddKnownTypes(bpav1alpha1.SchemeGroupVersion, provisioning, provisioning2, provisioning3) + + // Create Fake Clients and Clientset + fakeClient := fake.NewFakeClient(objs...) + fakeDyn := fakedynamic.NewSimpleDynamicClient(sc, bmhList) + fakeClientSet := fakeclientset.NewSimpleClientset() + + r := &ReconcileProvisioning{client: fakeClient, scheme: sc, clientset: fakeClientSet, bmhClient: fakeDyn} + + // Mock request to simulate Reconcile() being called on an event for a watched resource + req := simulateRequest(provisioning) + _, err = r.Reconcile(req) + if err != nil { + t.Fatalf("reconcile: (%v)", err) + } + + // Test 1: Check the job was created with the expected name + jobClient := r.clientset.BatchV1().Jobs(namespace) + job, err := jobClient.Get("kud-test-cluster", metav1.GetOptions{}) + + if err != nil { + t.Fatalf("Error occured while getting job: (%v)", err) + } + + // Test 2: Check that cluster name metadata in job is the expected cluster name + jobClusterName := job.Labels["cluster"] + if jobClusterName != clusterName { + t.Fatalf("Job cluster Name is wrong") + } + // Test 3: Check that the right error is produced when host with MAC address does not exist + req = simulateRequest(provisioning2) + _, err = r.Reconcile(req) + expectedErr := "Host with MAC Address " + macAddress2 + " not found\n" + if err.Error() != expectedErr { + t.Fatalf("Failed, Unexpected error occured %v\n", err) + } + + // Test 4: Check that the right error is produced when MAC address is not found in the DHCP lease file + req = simulateRequest(provisioning3) + _, err = r.Reconcile(req) + expectedErr = "IP address not found for host with MAC address " + macAddress3 + " \n" + if err.Error() != expectedErr { + t.Fatalf("Failed, Unexpected error occured %v\n", err) + } + + // Delete Fake DHCP file and cluster directories + err = os.Remove("/var/lib/dhcp/dhcpd.leases") + if err != nil { + t.Logf("\nUnable to delete fake DHCP file\n") + } + err = os.RemoveAll("/multi-cluster/" + clusterName) + if err != nil { + t.Logf("\nUnable to delete cluster directory %s\n", clusterName) + } + err = os.RemoveAll("/multi-cluster/" + clusterName2) + if err != nil { + t.Logf("\nUnable to delete cluster directory %s\n", clusterName2) + } + err = os.RemoveAll("/multi-cluster/" + clusterName3) + if err != nil { + t.Logf("\nUnable to delete cluster directory %s\n", clusterName3) + } } @@ -129,123 +126,107 @@ func simulateRequest(bpaCR *bpav1alpha1.Provisioning) reconcile.Request { return reconcile.Request{NamespacedName: namespacedName} } - - func newBPA(name, namespace, clusterName, macAddress string) *bpav1alpha1.Provisioning { - provisioningCR := &bpav1alpha1.Provisioning{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Labels: map[string]string{ - "cluster": clusterName, - }, - }, - Spec: bpav1alpha1.ProvisioningSpec{ - Masters: []map[string]bpav1alpha1.Master{ - map[string]bpav1alpha1.Master{ - "test-master" : bpav1alpha1.Master{ - MACaddress: macAddress, - }, - - }, - }, - }, - - } - return provisioningCR + provisioningCR := &bpav1alpha1.Provisioning{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{ + "cluster": clusterName, + }, + }, + Spec: bpav1alpha1.ProvisioningSpec{ + Masters: []map[string]bpav1alpha1.Master{ + map[string]bpav1alpha1.Master{ + "test-master": bpav1alpha1.Master{ + MACaddress: macAddress, + }, + }, + }, + }, + } + return provisioningCR } - -func newBMList() *unstructured.UnstructuredList{ +func newBMList() *unstructured.UnstructuredList { bmMap := map[string]interface{}{ - "apiVersion": "metal3.io/v1alpha1", - "kind": "BareMetalHostList", - "metaData": map[string]interface{}{ - "continue": "", - "resourceVersion": "11830058", - "selfLink": "/apis/metal3.io/v1alpha1/baremetalhosts", - - }, - } - - - + "apiVersion": "metal3.io/v1alpha1", + "kind": "BareMetalHostList", + "metaData": map[string]interface{}{ + "continue": "", + "resourceVersion": "11830058", + "selfLink": "/apis/metal3.io/v1alpha1/baremetalhosts", + }, + } metaData := map[string]interface{}{ - "creationTimestamp": "2019-10-24T04:51:15Z", - "generation":"1", - "name": "fake-test-bmh", - "namespace": "default", - "resourceVersion": "11829263", - "selfLink": "/apis/metal3.io/v1alpha1/namespaces/default/baremetalhosts/bpa-test-bmh", - "uid": "e92cb312-f619-11e9-90bc-00219ba0c77a", + "creationTimestamp": "2019-10-24T04:51:15Z", + "generation": "1", + "name": "fake-test-bmh", + "namespace": "default", + "resourceVersion": "11829263", + "selfLink": "/apis/metal3.io/v1alpha1/namespaces/default/baremetalhosts/bpa-test-bmh", + "uid": "e92cb312-f619-11e9-90bc-00219ba0c77a", } - - nicMap1 := map[string]interface{}{ - "ip": "", - "mac": "08:00:27:00:ab:2c", - "model": "0x8086 0x1572", - "name": "eth3", - "pxe": "false", - "speedGbps": "0", - "vlanId": "0", + "ip": "", + "mac": "08:00:27:00:ab:2c", + "model": "0x8086 0x1572", + "name": "eth3", + "pxe": "false", + "speedGbps": "0", + "vlanId": "0", } - nicMap2 := map[string]interface{}{ - "ip": "", - "mac": "08:00:27:00:ab:1c", - "model": "0x8086 0x1572", - "name": "eth4", - "pxe": "false", - "speedGbps": "0", - "vlanId": "0", - } - - specMap := map[string]interface{}{ - "status" : map[string]interface{}{ - "errorMessage": "", - "hardware": map[string]interface{}{ - "nics": map[string]interface{}{ - "nic1" : nicMap1, - "nic2" : nicMap2, - }, - }, - }, - + nicMap2 := map[string]interface{}{ + "ip": "", + "mac": "08:00:27:00:ab:1c", + "model": "0x8086 0x1572", + "name": "eth4", + "pxe": "false", + "speedGbps": "0", + "vlanId": "0", + } + specMap := map[string]interface{}{ + "status": map[string]interface{}{ + "errorMessage": "", + "hardware": map[string]interface{}{ + "nics": map[string]interface{}{ + "nic1": nicMap1, + "nic2": nicMap2, + }, + }, + }, } itemMap := map[string]interface{}{ - "apiVersion": "metal3.io/v1alpha1", - "kind": "BareMetalHost", - "metadata": metaData, - "spec": specMap, - } + "apiVersion": "metal3.io/v1alpha1", + "kind": "BareMetalHost", + "metadata": metaData, + "spec": specMap, + } itemU := unstructured.Unstructured{ - Object: itemMap, - } + Object: itemMap, + } - itemsList := []unstructured.Unstructured{itemU,} + itemsList := []unstructured.Unstructured{itemU} bmhList := &unstructured.UnstructuredList{ - Object: bmMap, - Items: itemsList, - } - + Object: bmMap, + Items: itemsList, + } - return bmhList + return bmhList } - // Create DHCP file for testing -func createFakeDHCP() error{ +func createFakeDHCP() error { - - dhcpData := []byte(`lease 192.168.50.63 { + dhcpData := []byte(`lease 192.168.50.63 { starts 4 2019/08/08 22:32:49; ends 4 2019/08/08 23:52:49; cltt 4 2019/08/08 22:32:49; @@ -255,11 +236,11 @@ func createFakeDHCP() error{ hardware ethernet 08:00:27:00:ab:2c; client-hostname "fake-test-bmh""; }`) - err := ioutil.WriteFile("/var/lib/dhcp/dhcpd.leases", dhcpData, 0777) + err := ioutil.WriteFile("/var/lib/dhcp/dhcpd.leases", dhcpData, 0777) - if (err != nil) { - return err - } + if err != nil { + return err + } - return nil + return nil }