X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=blobdiff_plain;f=cmd%2Fbpa-operator%2Fpkg%2Fcontroller%2Fprovisioning%2Fprovisioning_controller.go;h=b4e95779ecd7defa3cfd77f28763115c32ab9054;hb=refs%2Fchanges%2F65%2F1865%2F33;hp=dd13d88ec4df159c19700e22a8c9706fd3d0ba50;hpb=9bcb84340816a8815a3511f50105c4a2a5054bb1;p=icn.git diff --git a/cmd/bpa-operator/pkg/controller/provisioning/provisioning_controller.go b/cmd/bpa-operator/pkg/controller/provisioning/provisioning_controller.go index dd13d88..b4e9577 100644 --- a/cmd/bpa-operator/pkg/controller/provisioning/provisioning_controller.go +++ b/cmd/bpa-operator/pkg/controller/provisioning/provisioning_controller.go @@ -20,7 +20,6 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" - "k8s.io/client-go/rest" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" @@ -64,7 +63,23 @@ func Add(mgr manager.Manager) error { // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) reconcile.Reconciler { - return &ReconcileProvisioning{client: mgr.GetClient(), scheme: mgr.GetScheme()} + + config, err := config.GetConfig() + if err != nil { + fmt.Printf("Could not get kube config, Error: %v\n", err) + } + + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + fmt.Printf("Could not create clientset, Error: %v\n", err) + } + bmhDynamicClient, err := dynamic.NewForConfig(config) + + if err != nil { + fmt.Printf("Could not create dynamic client for bareMetalHosts, Error: %v\n", err) + } + + return &ReconcileProvisioning{client: mgr.GetClient(), scheme: mgr.GetScheme(), clientset: clientSet, bmhClient: bmhDynamicClient } } // add adds a new Controller to mgr with r as the reconcile.Reconciler @@ -120,6 +135,8 @@ type ReconcileProvisioning struct { // that reads objects from the cache and writes to the apiserver client client.Client scheme *runtime.Scheme + clientset kubernetes.Interface + bmhClient dynamic.Interface } // Reconcile reads that state of the cluster for a Provisioning object and makes changes based on the state read @@ -131,7 +148,6 @@ type ReconcileProvisioning struct { // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. func (r *ReconcileProvisioning) Reconcile(request reconcile.Request) (reconcile.Result, error) { reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) - //reqLogger.Info("Reconciling Provisioning") fmt.Printf("\n\n") reqLogger.Info("Reconciling Custom Resource") @@ -166,17 +182,6 @@ func (r *ReconcileProvisioning) Reconcile(request reconcile.Request) (reconcile. masterTag := "MASTER_" workerTag := "WORKER_" - config, err := config.GetConfig() - if err != nil { - fmt.Printf("Could not get kube config, Error: %v\n", err) - return reconcile.Result{}, err - } - - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - fmt.Printf("Could not create clientset, Error: %v\n", err) - return reconcile.Result{}, err - } if provisioningCreated { /////////////////////////////////////////////////////////////////////////////////////////////// @@ -186,34 +191,21 @@ func (r *ReconcileProvisioning) Reconcile(request reconcile.Request) (reconcile. clusterType := provisioningInstance.Labels["cluster-type"] mastersList := provisioningInstance.Spec.Masters workersList := provisioningInstance.Spec.Workers - dhcpLeaseFile := provisioningInstance.Spec.DHCPleaseFile - kudInstallerScript := provisioningInstance.Spec.KUDInstaller - multiClusterDir := provisioningInstance.Spec.MultiClusterPath + kudPlugins := provisioningInstance.Spec.KUDPlugins - bareMetalHostList, _ := listBareMetalHosts(config) - virtletVMList, _ := listVirtletVMs() + bareMetalHostList, _ := listBareMetalHosts(r.bmhClient) + virtletVMList, _ := listVirtletVMs(r.clientset) - var allString string - var masterString string - var workerString string - defaultDHCPFile := "/var/lib/dhcp/dhcpd.leases" - defaultKUDInstallerPath := "/multicloud-k8s/kud/hosting_providers/vagrant" - defaultMultiClusterDir := "/multi-cluster" - //Give Default values for paths if no path is given in the CR - if dhcpLeaseFile == "" { - dhcpLeaseFile = defaultDHCPFile - } - if kudInstallerScript == "" { - kudInstallerScript = defaultKUDInstallerPath - } + var allString string + var masterString string + var workerString string - if multiClusterDir == "" { - multiClusterDir = defaultMultiClusterDir - } + dhcpLeaseFile := "/var/lib/dhcp/dhcpd.leases" + multiClusterDir := "/multi-cluster" //Create Directory for the specific cluster clusterDir := multiClusterDir + "/" + clusterName @@ -265,6 +257,9 @@ func (r *ReconcileProvisioning) Reconcile(request reconcile.Request) (reconcile. } allString += masterLabel + " ansible_ssh_host=" + hostIPaddress + " ansible_ssh_port=22" + "\n" + if clusterType == "virtlet-vm" { + allString = masterLabel + " ansible_ssh_host=" + hostIPaddress + " ansible_ssh_port=22" + " ansible_ssh_user=root" + " ansible_ssh_pass=root" + "\n" + } masterString += masterLabel + "\n" clusterData[masterTag + masterLabel] = hostIPaddress @@ -345,8 +340,10 @@ func (r *ReconcileProvisioning) Reconcile(request reconcile.Request) (reconcile. } fmt.Printf("%s : %s \n", hostIPaddress, workerMAC) - allString += workerLabel + " ansible_ssh_host=" + hostIPaddress + " ansible_ssh_port=22" + "\n" + if clusterType == "virtlet-vm" { + allString = masterLabel + " ansible_ssh_host=" + hostIPaddress + " ansible_ssh_port=22" + " ansible_ssh_user=root" + " ansible_ssh_pass=root" + "\n" + } workerString += workerLabel + "\n" clusterData[workerTag + workerLabel] = hostIPaddress @@ -417,17 +414,22 @@ func (r *ReconcileProvisioning) Reconcile(request reconcile.Request) (reconcile. return reconcile.Result{}, err } - if clusterType == "virtlet-vm" { - _, err = hostFile.NewRawSection("ovn-central", masterString) - if err != nil { - fmt.Printf("Error occured while creating section \n %v", err) - return reconcile.Result{}, err - } - _, err = hostFile.NewRawSection("ovn-controller", masterString) - if err != nil { - fmt.Printf("Error occured while creating section \n %v", err) - return reconcile.Result{}, err - } + _, err = hostFile.NewRawSection("ovn-central", masterString) + if err != nil { + fmt.Printf("Error occured while creating section \n %v", err) + return reconcile.Result{}, err + } + + _, err = hostFile.NewRawSection("ovn-controller", workerString) + if err != nil { + fmt.Printf("Error occured while creating section \n %v", err) + return reconcile.Result{}, err + } + + _, err = hostFile.NewRawSection("virtlet", workerString) + if err != nil { + fmt.Printf("Error occured while creating section \n %v", err) + return reconcile.Result{}, err } _, err = hostFile.NewRawSection("k8s-cluster:children", "kube-node\n" + "kube-master") @@ -441,7 +443,7 @@ func (r *ReconcileProvisioning) Reconcile(request reconcile.Request) (reconcile. hostFile.SaveTo(iniHostFilePath) //Install KUD - err = createKUDinstallerJob(clusterName, request.Namespace, clusterLabel, clientset) + err = createKUDinstallerJob(clusterName, request.Namespace, clusterLabel, kudPlugins, r.clientset) if err != nil { fmt.Printf("Error occured while creating KUD Installer job for cluster %v\n ERROR: %v", clusterName, err) return reconcile.Result{}, err @@ -449,7 +451,7 @@ func (r *ReconcileProvisioning) Reconcile(request reconcile.Request) (reconcile. //Start separate thread to keep checking job status, Create an IP address configmap //for cluster if KUD is successfully installed - go checkJob(clusterName, request.Namespace, clusterData, clusterLabel, clientset) + go checkJob(clusterName, request.Namespace, clusterData, clusterLabel, r.clientset) return reconcile.Result{}, nil @@ -464,7 +466,7 @@ func (r *ReconcileProvisioning) Reconcile(request reconcile.Request) (reconcile. defaultSSHPrivateKey := "/root/.ssh/id_rsa" //Get IP address configmap for the cluster - clusterConfigMapData, err := getConfigMapData(request.Namespace, softwareClusterName, clientset) + clusterConfigMapData, err := getConfigMapData(request.Namespace, softwareClusterName, r.clientset) if err != nil { fmt.Printf("Error occured while retrieving IP address Data for cluster %s, ERROR: %v\n", softwareClusterName, err) return reconcile.Result{}, err @@ -493,15 +495,7 @@ func (r *ReconcileProvisioning) Reconcile(request reconcile.Request) (reconcile. } //Function to Get List containing baremetal hosts -func listBareMetalHosts(config *rest.Config) (*unstructured.UnstructuredList, error) { - - //Create Dynamic Client for BareMetalHost CRD - bmhDynamicClient, err := dynamic.NewForConfig(config) - - if err != nil { - fmt.Println("Could not create dynamic client for bareMetalHosts, Error: %v\n", err) - return &unstructured.UnstructuredList{}, err - } +func listBareMetalHosts(bmhDynamicClient dynamic.Interface) (*unstructured.UnstructuredList, error) { //Create GVR representing a BareMetalHost CR bmhGVR := schema.GroupVersionResource{ @@ -513,7 +507,7 @@ func listBareMetalHosts(config *rest.Config) (*unstructured.UnstructuredList, er //Get List containing all BareMetalHosts CRs bareMetalHosts, err := bmhDynamicClient.Resource(bmhGVR).List(metav1.ListOptions{}) if err != nil { - fmt.Println("Error occured, cannot get BareMetalHosts list, Error: %v\n", err) + fmt.Printf("Error occured, cannot get BareMetalHosts list, Error: %v\n", err) return &unstructured.UnstructuredList{}, err } @@ -549,7 +543,7 @@ func getHostIPaddress(macAddress string, dhcpLeaseFilePath string ) (string, err //Read the dhcp lease file dhcpFile, err := ioutil.ReadFile(dhcpLeaseFilePath) if err != nil { - fmt.Println("Failed to read lease file\n") + fmt.Printf("Failed to read lease file\n") return "", err } @@ -559,7 +553,7 @@ func getHostIPaddress(macAddress string, dhcpLeaseFilePath string ) (string, err reg := "lease.*{|ethernet.*|\n. binding state.*" re, err := regexp.Compile(reg) if err != nil { - fmt.Println("Could not create Regexp object, Error %v occured\n", err) + fmt.Printf("Could not create Regexp object, Error %v occured\n", err) return "", err } @@ -592,7 +586,7 @@ func getHostIPaddress(macAddress string, dhcpLeaseFilePath string ) (string, err } //Function to create configmap -func createConfigMap(data, labels map[string]string, namespace string, clientset *kubernetes.Clientset) error{ +func createConfigMap(data, labels map[string]string, namespace string, clientset kubernetes.Interface) error{ configmapClient := clientset.CoreV1().ConfigMaps(namespace) @@ -616,7 +610,7 @@ func createConfigMap(data, labels map[string]string, namespace string, clientset } //Function to get configmap Data -func getConfigMapData(namespace, clusterName string, clientset *kubernetes.Clientset) (map[string]string, error) { +func getConfigMapData(namespace, clusterName string, clientset kubernetes.Interface) (map[string]string, error) { configmapClient := clientset.CoreV1().ConfigMaps(namespace) configmapName := clusterName + "-configmap" @@ -630,11 +624,24 @@ func getConfigMapData(namespace, clusterName string, clientset *kubernetes.Clien } //Function to create job for KUD installation -func createKUDinstallerJob(clusterName, namespace string, labels map[string]string, clientset *kubernetes.Clientset) error{ +func createKUDinstallerJob(clusterName, namespace string, labels map[string]string, kudPlugins []string, clientset kubernetes.Interface) error{ var backOffLimit int32 = 0 var privi bool = true + installerString := " ./installer --cluster " + clusterName + + // Check if any plugin was specified + if len(kudPlugins) > 0 { + plugins := " --plugins" + + for _, plug := range kudPlugins { + plugins += " " + plug + } + + installerString += plugins + } + jobClient := clientset.BatchV1().Jobs("default") @@ -668,7 +675,7 @@ func createKUDinstallerJob(clusterName, namespace string, labels map[string]stri }, Command: []string{"/bin/sh","-c"}, - Args: []string{"cp -r /.ssh /root/; chmod -R 600 /root/.ssh; ./installer --cluster " + clusterName}, + Args: []string{"cp -r /.ssh /root/; chmod -R 600 /root/.ssh;" + installerString}, SecurityContext: &corev1.SecurityContext{ Privileged : &privi, @@ -707,7 +714,7 @@ func createKUDinstallerJob(clusterName, namespace string, labels map[string]stri } //Function to Check if job succeeded -func checkJob(clusterName, namespace string, data, labels map[string]string, clientset *kubernetes.Clientset) { +func checkJob(clusterName, namespace string, data, labels map[string]string, clientset kubernetes.Interface) { fmt.Printf("\nChecking job status for cluster %s\n", clusterName) jobName := "kud-" + clusterName @@ -837,26 +844,13 @@ func sshInstaller(softwareString, sshPrivateKey, ipAddress string) error { } -func listVirtletVMs() ([]VirtletVM, error) { +func listVirtletVMs(clientset kubernetes.Interface) ([]VirtletVM, error) { var vmPodList []VirtletVM - config, err := config.GetConfig() - if err != nil { - fmt.Println("Could not get kube config, Error: %v\n", err) - return []VirtletVM{}, err - } - - // create the clientset - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - fmt.Println("Could not create the client set, Error: %v\n", err) - return []VirtletVM{}, err - } - pods, err := clientset.CoreV1().Pods("").List(metav1.ListOptions{}) if err != nil { - fmt.Println("Could not get pod info, Error: %v\n", err) + fmt.Printf("Could not get pod info, Error: %v\n", err) return []VirtletVM{}, err } @@ -867,7 +861,7 @@ func listVirtletVMs() ([]VirtletVM, error) { annotation, err := json.Marshal(pod.ObjectMeta.GetAnnotations()) if err != nil { - fmt.Println("Could not get pod annotations, Error: %v\n", err) + fmt.Printf("Could not get pod annotations, Error: %v\n", err) return []VirtletVM{}, err } @@ -878,8 +872,8 @@ func listVirtletVMs() ([]VirtletVM, error) { podStatusJson, _ := json.Marshal(pod.Status) json.Unmarshal([]byte(podStatusJson), &podStatus) - if runtime == "virtlet.cloud" && podStatus.Phase == "Running" && podAnnotation["v1.multus-cni.io/default-network"] != nil { - ns := podAnnotation["v1.multus-cni.io/default-network"].(string) + if runtime == "virtlet.cloud" && podStatus.Phase == "Running" && podAnnotation["k8s.v1.cni.cncf.io/networks-status"] != nil { + ns := podAnnotation["k8s.v1.cni.cncf.io/networks-status"].(string) json.Unmarshal([]byte(ns), &podDefaultNetStatus) vmPodList = append(vmPodList, VirtletVM{podStatus.PodIP, podDefaultNetStatus[0].Mac})