From: Andrew Bays Date: Tue, 8 Oct 2019 10:57:59 +0000 (-0400) Subject: Automated masters and workers deploy X-Git-Tag: akraino_r2~19 X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F46%2F1746%2F3;p=kni%2Finstaller.git Automated masters and workers deploy Change-Id: I9ed643e4fee34fef6c73bbf0a5266c9cfba401db --- diff --git a/cmd/deploy_masters.go b/cmd/deploy_masters.go new file mode 100644 index 0000000..6683ae3 --- /dev/null +++ b/cmd/deploy_masters.go @@ -0,0 +1,60 @@ +// Copyright © 2019 Red Hat +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "log" + "os" + + "gerrit.akraino.org/kni/installer/pkg/site" + "github.com/spf13/cobra" +) + +// deployMastersCmd represents the automate_masters_deployment command +var deployMastersCmd = &cobra.Command{ + Use: "deploy_masters siteName [--build_path=]", + Short: "Command to automate the deployment of the master nodes of a previously-prepared site", + Long: ``, + TraverseChildren: true, + Run: func(cmd *cobra.Command, args []string) { + // retrieve config values and start fetching + var siteName string + if len(args) == 0 { + log.Fatal("Please specify site name as first argument") + os.Exit(1) + } else { + siteName = args[0] + } + + buildPath, _ := cmd.Flags().GetString("build_path") + if len(buildPath) == 0 { + // will generate a temporary directory + buildPath = fmt.Sprintf("%s/.kni", os.Getenv("HOME")) + } + + // This command is used after fetch_requirements and prepare_manifests, + // so the site directory should be available on disk already (if not, + // s.AutomateMastersDeployment will error-out appropriately) + s := site.NewWithName(siteName, buildPath) + s.AutomateMastersDeployment() + }, +} + +func init() { + rootCmd.AddCommand(deployMastersCmd) + + deployMastersCmd.Flags().StringP("build_path", "", "", "Directory to use as build path. If that doesn't exist, the installer will generate a default directory") +} diff --git a/cmd/deploy_workers.go b/cmd/deploy_workers.go new file mode 100644 index 0000000..20b9881 --- /dev/null +++ b/cmd/deploy_workers.go @@ -0,0 +1,60 @@ +// Copyright © 2019 Red Hat +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "log" + "os" + + "gerrit.akraino.org/kni/installer/pkg/site" + "github.com/spf13/cobra" +) + +// deployWorkersCmd represents the automate_workers_deployment command +var deployWorkersCmd = &cobra.Command{ + Use: "deploy_workers siteName [--build_path=]", + Short: "Command to automate the deployment of the worker nodes of a previously-prepared site", + Long: ``, + TraverseChildren: true, + Run: func(cmd *cobra.Command, args []string) { + // retrieve config values and start fetching + var siteName string + if len(args) == 0 { + log.Fatal("Please specify site name as first argument") + os.Exit(1) + } else { + siteName = args[0] + } + + buildPath, _ := cmd.Flags().GetString("build_path") + if len(buildPath) == 0 { + // will generate a temporary directory + buildPath = fmt.Sprintf("%s/.kni", os.Getenv("HOME")) + } + + // This command is used after fetch_requirements, prepare_manifests + // and deploy_masters, so the site directory and required automation + // configs are already available on disk + s := site.NewWithName(siteName, buildPath) + s.AutomateWorkersDeployment() + }, +} + +func init() { + rootCmd.AddCommand(deployWorkersCmd) + + deployWorkersCmd.Flags().StringP("build_path", "", "", "Directory to use as build path. If that doesn't exist, the installer will generate a default directory") +} diff --git a/pkg/automation/automation.go b/pkg/automation/automation.go index 559743c..3fdf891 100644 --- a/pkg/automation/automation.go +++ b/pkg/automation/automation.go @@ -5,6 +5,16 @@ import ( "fmt" ) +// ProfileType, SiteBuildPath and SiteName are always needed (and are always +// checked by the automation package's New function). Anything else added here +// is the responsibility of the automation interface implementation to validate +type AutomatedDeploymentParams struct { + ProfileType string + SiteBuildPath string + SiteName string + SiteRepo string +} + type AutomatedDeploymentInterface interface { PrepareBastion() error // Prepare host for automation DeployMasters() error // Deploy cluster masters @@ -15,39 +25,38 @@ var ( // If we find that different profile types (libvirt, aws, etc) that we add // in the future require different constructor parameter count/types, then // we can redo the approach here - automatedDeploymentConstructors map[string]func(string, string, string) (AutomatedDeploymentInterface, error) + automatedDeploymentConstructors map[string]func(AutomatedDeploymentParams) (AutomatedDeploymentInterface, error) ) func init() { // Add new automation profile types here - automatedDeploymentConstructors = map[string]func(string, string, string) (AutomatedDeploymentInterface, error){} + automatedDeploymentConstructors = map[string]func(AutomatedDeploymentParams) (AutomatedDeploymentInterface, error){} automatedDeploymentConstructors["baremetal"] = newBaremetal } // Generates a new automation deployment instance -func New(profileType string, siteBuildPath string, siteName string, siteRepo string) (AutomatedDeploymentInterface, error) { - if siteBuildPath == "" { +func New(params AutomatedDeploymentParams) (AutomatedDeploymentInterface, error) { + // SiteBuildPath is always needed + if params.SiteBuildPath == "" { return nil, errors.New("AutomatedDeployment: New: site build path not provided") } - if siteName == "" { + // SiteName is always needed + if params.SiteName == "" { return nil, errors.New("AutomatedDeployment: New: site name not provided") } - if siteRepo == "" { - return nil, errors.New("AutomatedDeployment: New: site repo not provided") - } - - constructor := automatedDeploymentConstructors[profileType] + // ProfileType is always needed + constructor := automatedDeploymentConstructors[params.ProfileType] // If no constructor available, then automation is not available for this profile type if constructor == nil { - return nil, fmt.Errorf("AutomatedDeployment: New: automation not supported for profile type '%s", profileType) + return nil, fmt.Errorf("AutomatedDeployment: New: automation not supported for profile type '%s", params.ProfileType) } // Constructors should return nil as the AutomatedDeploymentInterface if automation is // not supported for the particular site - automatedDeployment, err := constructor(siteBuildPath, siteName, siteRepo) + automatedDeployment, err := constructor(params) if err != nil { return nil, err diff --git a/pkg/automation/baremetal.go b/pkg/automation/baremetal.go index deaf08a..7939369 100644 --- a/pkg/automation/baremetal.go +++ b/pkg/automation/baremetal.go @@ -9,6 +9,8 @@ import ( "os/exec" "path/filepath" + "github.com/otiai10/copy" + getter "github.com/hashicorp/go-getter" yaml "gopkg.in/yaml.v2" ) @@ -27,9 +29,15 @@ type baremetalAutomatedDeployment struct { siteRepo string } -func newBaremetal(siteBuildPath string, siteName string, siteRepo string) (AutomatedDeploymentInterface, error) { +type scriptRunInstance struct { + description string + scriptFile string + args []string +} + +func newBaremetal(params AutomatedDeploymentParams) (AutomatedDeploymentInterface, error) { // Examine site's site-config and determine if automation is even possible for this site - siteConfigSourcePath := fmt.Sprintf("%s/%s/site/00_install-config/site-config.yaml", siteBuildPath, siteName) + siteConfigSourcePath := fmt.Sprintf("%s/%s/site/00_install-config/site-config.yaml", params.SiteBuildPath, params.SiteName) filename, _ := filepath.Abs(siteConfigSourcePath) siteConfigFile, err := ioutil.ReadFile(filename) @@ -53,9 +61,9 @@ func newBaremetal(siteBuildPath string, siteName string, siteRepo string) (Autom } return baremetalAutomatedDeployment{ - siteBuildPath: siteBuildPath, - siteName: siteName, - siteRepo: siteRepo, + siteBuildPath: params.SiteBuildPath, + siteName: params.SiteName, + siteRepo: params.SiteRepo, }, nil } @@ -122,11 +130,293 @@ func (bad baremetalAutomatedDeployment) PrepareBastion() error { } func (bad baremetalAutomatedDeployment) DeployMasters() error { - // TODO + sitePath := fmt.Sprintf("%s/%s", bad.siteBuildPath, bad.siteName) + + // Make sure final_manifests directory is available + finalManifestsPath := fmt.Sprintf("%s/final_manifests", sitePath) + + _, err := os.Stat(finalManifestsPath) + + if err != nil { + return fmt.Errorf("baremetalAutomatedDeployment: DeployMasters: unable to access final manifests at %s: %s", finalManifestsPath, err) + } + + // Make sure automation-required manifests are available (these YAMLs should have been copied + // to the directory during prepare_manifests) + automationManifestsPath := fmt.Sprintf("%s/automation", sitePath) + + _, err = os.Stat(automationManifestsPath) + + if err != nil { + return fmt.Errorf("baremetalAutomatedDeployment: DeployMasters: unable to access automation manifests at %s: %s", automationManifestsPath, err) + } + + // Make sure the baremetal automation repo is locally available + automationRepoPath := fmt.Sprintf("%s/baremetal_automation", sitePath) + + _, err = os.Stat(automationRepoPath) + + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("baremetalAutomatedDeployment: DeployMasters: unable to access local automation repo at %s: %s", automationRepoPath, err) + } + + // Doesn't exist, so clone it? + // NOTE: It should already exist, having been created during the "fetch_requirements" step + log.Printf("baremetalAutomatedDeployment: DeployMasters: downloading missing baremetal automation repo (%s)\n", automationRemoteSource) + + client := &getter.Client{Src: automationRemoteSource, Dst: automationRepoPath, Mode: getter.ClientModeAny} + err := client.Get() + + if err != nil { + return fmt.Errorf("baremetalAutomatedDeployment: DeployMasters: error cloning baremetal automation repository: %s", err) + } + } + + // Copy final_manifests into the automation repo's ocp directory (the ocp + // directory is the default location that the automation scripts use for + // various openshift-install calls) + err = copy.Copy(finalManifestsPath, fmt.Sprintf("%s/ocp", automationRepoPath)) + + if err != nil { + return fmt.Errorf("baremetalAutomatedDeployment: DeployMasters: error copying final_manifests into automation ocp directory: %s", err) + } + + // Now run the actual automation scripts + err = bad.runConfigGenerationScripts(automationRepoPath, automationManifestsPath) + + if err != nil { + return err + } + + // Then start the containers + err = bad.runContainers(automationRepoPath) + + if err != nil { + return err + } + + // Finally run terraform commands to begin cluster deployment + err = bad.runTerraform(automationRepoPath, "cluster") + + if err != nil { + return err + } + + log.Printf("baremetalAutomatedDeployment: DeployMasters: bootstrap and master(s) deploy initiated...\n") + + return nil +} + +// automationRepoPath: contains path to automation repo directory +func (bad baremetalAutomatedDeployment) runContainers(automationRepoPath string) error { + // Add scripts to run + scripts := []scriptRunInstance{} + + scripts = append(scripts, scriptRunInstance{ + description: "dnsmasq provisioning container start", + scriptFile: "gen_config_prov.sh", + args: []string{"start"}, + }) + + scripts = append(scripts, scriptRunInstance{ + description: "dnsmasq baremetal container start", + scriptFile: "gen_config_bm.sh", + args: []string{"start"}, + }) + + // Need to make sure haproxy is built before running + scripts = append(scripts, scriptRunInstance{ + description: "haproxy container build", + scriptFile: "gen_haproxy.sh", + args: []string{"build"}, + }) + + scripts = append(scripts, scriptRunInstance{ + description: "haproxy container start", + scriptFile: "gen_haproxy.sh", + args: []string{"start"}, + }) + + scripts = append(scripts, scriptRunInstance{ + description: "coredns container start", + scriptFile: "gen_coredns.sh", + args: []string{"start"}, + }) + + scripts = append(scripts, scriptRunInstance{ + description: "matchbox container start", + scriptFile: "gen_matchbox.sh", + args: []string{"start"}, + }) + + log.Printf("baremetalAutomatedDeployment: runContainers: starting bastion containers...\n") + + err := bad.runScripts(automationRepoPath, scripts) + + if err != nil { + return err + } + + log.Printf("baremetalAutomatedDeployment: runContainers: bastion containers successfully started\n") + + return nil +} + +// automationRepoPath: contains path to automation repo directory +// automationManifestsPath: contains path to directory containing site-config.yaml, install-config.yaml +// and any required credential secret yamls +func (bad baremetalAutomatedDeployment) runConfigGenerationScripts(automationRepoPath string, automationManifestsPath string) error { + // Add scripts to run + scripts := []scriptRunInstance{} + + commonArgs := []string{ + fmt.Sprintf("-m%s", automationManifestsPath), + } + + scripts = append(scripts, scriptRunInstance{ + description: "dnsmasq provisioning config generation", + scriptFile: "gen_config_prov.sh", + args: commonArgs, + }) + + scripts = append(scripts, scriptRunInstance{ + description: "dnsmasq baremetal config generation", + scriptFile: "gen_config_bm.sh", + args: commonArgs, + }) + + scripts = append(scripts, scriptRunInstance{ + description: "coredns config generation", + scriptFile: "gen_coredns.sh", + args: append([]string{"all"}, commonArgs...), + }) + + scripts = append(scripts, scriptRunInstance{ + description: "haproxy config generation", + scriptFile: "gen_haproxy.sh", + args: append(commonArgs, "gen-config"), + }) + + scripts = append(scripts, scriptRunInstance{ + description: "matchbox repo generation", + scriptFile: "gen_matchbox.sh", + args: append([]string{"repo"}, commonArgs...), + }) + + scripts = append(scripts, scriptRunInstance{ + description: "matchbox data generation", + scriptFile: "gen_matchbox.sh", + args: append([]string{"data"}, commonArgs...), + }) + + scripts = append(scripts, scriptRunInstance{ + description: "terraform cluster/work config generation", + scriptFile: "gen_terraform.sh", + args: append([]string{"all"}, commonArgs...), + }) + + scripts = append(scripts, scriptRunInstance{ + description: "terraform installation", + scriptFile: "gen_terraform.sh", + args: append([]string{"install"}, commonArgs...), + }) + + scripts = append(scripts, scriptRunInstance{ + description: "ignition config generation", + scriptFile: "gen_ignition.sh", + args: append([]string{"create-output"}, commonArgs...), + }) + + log.Printf("baremetalAutomatedDeployment: runConfigGenerationScripts: generating configuration...\n") + + err := bad.runScripts(automationRepoPath, scripts) + + if err != nil { + return err + } + + log.Printf("baremetalAutomatedDeployment: runConfigGenerationScripts: configuration successfully generated\n") + return nil } func (bad baremetalAutomatedDeployment) DeployWorkers() error { - // TODO + sitePath := fmt.Sprintf("%s/%s", bad.siteBuildPath, bad.siteName) + automationRepoPath := fmt.Sprintf("%s/baremetal_automation", sitePath) + + _, err := os.Stat(automationRepoPath) + + if err != nil { + return fmt.Errorf("baremetalAutomatedDeployment: DeployWorkers: unable to access local automation repo at %s: %s", automationRepoPath, err) + } + + // Finally run terraform commands to begin workers deployment + err = bad.runTerraform(automationRepoPath, "workers") + + if err != nil { + return err + } + + log.Printf("baremetalAutomatedDeployment: DeployWorkers: worker(s) deploy initiated...\n") + + return nil +} + +func (bad baremetalAutomatedDeployment) runTerraform(automationRepoPath string, targetType string) error { + terraformPath := fmt.Sprintf("%s/terraform/%s", automationRepoPath, targetType) + + log.Printf("baremetalAutomatedDeployment: runTerraform: initializing terraform...\n") + + // Init + cmd := exec.Command("terraform", "init") + cmd.Dir = terraformPath + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Run() + + if err != nil { + return fmt.Errorf("baremetalAutomatedDeployment: runTerraform: error running baremetal automation %s terraform init: %s", targetType, err) + } + + log.Printf("baremetalAutomatedDeployment: runTerraform: terraform successfully initialized\n") + log.Printf("baremetalAutomatedDeployment: runTerraform: applying terraform...\n") + + // Apply + cmd = exec.Command("terraform", "apply", "--auto-approve") + cmd.Dir = terraformPath + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err = cmd.Run() + + if err != nil { + return fmt.Errorf("baremetalAutomatedDeployment: runTerraform: error running baremetal automation %s terraform apply: %s", targetType, err) + } + + log.Printf("baremetalAutomatedDeployment: runTerraform: terraform successfully applied\n") + + return nil +} + +func (bad baremetalAutomatedDeployment) runScripts(automationRepoPath string, scripts []scriptRunInstance) error { + for _, script := range scripts { + cmd := exec.Command(fmt.Sprintf("%s/scripts/%s", automationRepoPath, script.scriptFile), script.args...) + cmd.Dir = automationRepoPath + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + log.Printf("baremetalAutomatedDeployment: runScripts: running %s script...\n", script.description) + + err := cmd.Run() + + if err != nil { + return fmt.Errorf("baremetalAutomatedDeployment: runScripts: error running %s script: %s", script.description, err) + } + + log.Printf("baremetalAutomatedDeployment: runScripts: finished running %s script\n", script.description) + } + return nil } diff --git a/pkg/site/site.go b/pkg/site/site.go index 8d124aa..6e788a6 100644 --- a/pkg/site/site.go +++ b/pkg/site/site.go @@ -341,7 +341,8 @@ func (s Site) PrepareManifests() { automationPath := fmt.Sprintf("%s/automation", sitePath) os.Mkdir(automationPath, 0755) - // copy 00_install-config directory contents into automation sub-directory + // copy 00_install-config directory contents (minus kustomization.yaml) + // into automation sub-directory installConfigDirPath := fmt.Sprintf("%s/blueprint/sites/site/00_install-config", sitePath) err := copy.Copy(installConfigDirPath, automationPath) @@ -350,6 +351,10 @@ func (s Site) PrepareManifests() { os.Exit(1) } + // Remove kustomization from automation sub-directory (the copy library used + // above does not allow for filtering files when copying directories) + os.Remove(fmt.Sprintf("%s/kustomization.yaml", automationPath)) + // generate openshift-install manifests based on phase 00_install-config assetsPath := fmt.Sprintf("%s/generated_assets", sitePath) os.RemoveAll(assetsPath) @@ -463,6 +468,73 @@ func (s Site) ApplyWorkloads(kubeconfigFile string) { } } +func (s Site) AutomateMastersDeployment() { + // Run the automated deployment + err := s.automateDeployment("masters") + + if err != nil { + log.Fatal(fmt.Sprintf("Site: AutomateMastersDeployment: Error attempting to run automated deployment: %s", err)) + os.Exit(1) + } +} + +func (s Site) AutomateWorkersDeployment() { + // Run the automated deployment + err := s.automateDeployment("workers") + + if err != nil { + log.Fatal(fmt.Sprintf("Site: AutomateWorkersDeployment: Error attempting to run automated deployment: %s", err)) + os.Exit(1) + } +} + +func (s Site) automateDeployment(deploymentType string) error { + // Get profile name + profileName, _, _ := s.GetProfileFromSite() + + // Get the profile type + // NOTE: This also checks whether the site repo exists locally, so there is no + // need to check that here + profileType, err := s.getProfileType(profileName) + + if err != nil { + log.Fatal(fmt.Sprintf("Site: automateDeployment: Error acquiring site profile type: %s", err)) + os.Exit(1) + } + + // Create an automated deployment instance + automatedDeploymentParams := automation.AutomatedDeploymentParams{ + ProfileType: profileType, + SiteBuildPath: s.buildPath, + SiteName: s.siteName, + SiteRepo: s.siteRepo, + } + + automatedDeployment, err := automation.New(automatedDeploymentParams) + + if err != nil { + log.Fatal(fmt.Sprintf("Site: automateDeployment: Error creating automated deployment instance: %s", err)) + os.Exit(1) + } + + // If nil is returned for automatedDeployment, then this particular site does + // not contain the necessary config required to automate its deployment + if automatedDeployment == nil { + return fmt.Errorf("Site: automateDeployment: automated deployment not supported for site '%s'", s.siteName) + } + + switch deploymentType { + case "masters": + return automatedDeployment.DeployMasters() + case "workers": + return automatedDeployment.DeployWorkers() + default: + return fmt.Errorf("Site: automateDeployment: unknown deployment type: %s", deploymentType) + } + + return nil +} + // Determines site profile type based on blueprint profile contents func (s Site) getProfileType(profileName string) (string, error) { if profileName == "" || s.buildPath == "" || s.siteName == "" { @@ -476,7 +548,14 @@ func (s Site) getProfileType(profileName string) (string, error) { _, err := os.Stat(installConfigDirPath) if err != nil { - return "", fmt.Errorf("Site: getProfileType: blueprint profile install config directory (%s) not found", installConfigDirPath) + // Check the other possible location + installConfigDirPath = fmt.Sprintf("%s/blueprint/profiles/%s/00_install-config", sitePath, profileName) + + _, err := os.Stat(installConfigDirPath) + + if err != nil { + return "", fmt.Errorf("Site: getProfileType: blueprint profile install config directory (%s) not found", installConfigDirPath) + } } var profileType string @@ -570,7 +649,14 @@ func (s Site) prepareHostForAutomation(profileName string) error { } // Attempt to create an automated deployment instance - automatedDeployment, err := automation.New(profileType, s.buildPath, s.siteName, s.siteRepo) + automatedDeploymentParams := automation.AutomatedDeploymentParams{ + ProfileType: profileType, + SiteBuildPath: s.buildPath, + SiteName: s.siteName, + SiteRepo: s.siteRepo, + } + + automatedDeployment, err := automation.New(automatedDeploymentParams) if err != nil { // If automation isn't supported for this profile type, it's not a fatal error in