From: Andrew Bays Date: Fri, 11 Oct 2019 19:32:57 +0000 (-0400) Subject: Automated cluster teardown command X-Git-Tag: akraino_r2~18 X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F74%2F1774%2F1;p=kni%2Finstaller.git Automated cluster teardown command Change-Id: Id4d4df957b08a1b38b50fbe8bd948904f82a936f --- diff --git a/cmd/deploy_masters.go b/cmd/deploy_masters.go index 6683ae3..7194978 100644 --- a/cmd/deploy_masters.go +++ b/cmd/deploy_masters.go @@ -23,7 +23,7 @@ import ( "github.com/spf13/cobra" ) -// deployMastersCmd represents the automate_masters_deployment command +// deployMastersCmd represents the deploy_masters command var deployMastersCmd = &cobra.Command{ Use: "deploy_masters siteName [--build_path=]", Short: "Command to automate the deployment of the master nodes of a previously-prepared site", diff --git a/cmd/deploy_workers.go b/cmd/deploy_workers.go index 20b9881..c547303 100644 --- a/cmd/deploy_workers.go +++ b/cmd/deploy_workers.go @@ -23,7 +23,7 @@ import ( "github.com/spf13/cobra" ) -// deployWorkersCmd represents the automate_workers_deployment command +// deployWorkersCmd represents the deploy_workers command var deployWorkersCmd = &cobra.Command{ Use: "deploy_workers siteName [--build_path=]", Short: "Command to automate the deployment of the worker nodes of a previously-prepared site", diff --git a/cmd/destroy_cluster.go b/cmd/destroy_cluster.go new file mode 100644 index 0000000..747372f --- /dev/null +++ b/cmd/destroy_cluster.go @@ -0,0 +1,60 @@ +// Copyright © 2019 Red Hat +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "log" + "os" + + "gerrit.akraino.org/kni/installer/pkg/site" + "github.com/spf13/cobra" +) + +// destroyClusterCmd represents the destroy_cluster command +var destroyClusterCmd = &cobra.Command{ + Use: "destroy_cluster siteName [--build_path=]", + Short: "Command to automate the teardown of master and workers nodes of an automated-deployment cluster", + Long: ``, + TraverseChildren: true, + Run: func(cmd *cobra.Command, args []string) { + // retrieve config values and start fetching + var siteName string + if len(args) == 0 { + log.Fatal("Please specify site name as first argument") + os.Exit(1) + } else { + siteName = args[0] + } + + buildPath, _ := cmd.Flags().GetString("build_path") + if len(buildPath) == 0 { + // will generate a temporary directory + buildPath = fmt.Sprintf("%s/.kni", os.Getenv("HOME")) + } + + // This command is used after fetch_requirements and prepare_manifests, + // so the site directory should be available on disk already (if not, + // s.AutomateMastersDeployment will error-out appropriately) + s := site.NewWithName(siteName, buildPath) + s.AutomateClusterDestroy() + }, +} + +func init() { + rootCmd.AddCommand(destroyClusterCmd) + + destroyClusterCmd.Flags().StringP("build_path", "", "", "Directory to use as build path. If that doesn't exist, the installer will generate a default directory") +} diff --git a/pkg/automation/automation.go b/pkg/automation/automation.go index 3fdf891..e5c4304 100644 --- a/pkg/automation/automation.go +++ b/pkg/automation/automation.go @@ -19,6 +19,7 @@ type AutomatedDeploymentInterface interface { PrepareBastion() error // Prepare host for automation DeployMasters() error // Deploy cluster masters DeployWorkers() error // Deploy cluster workers + DestroyCluster() error // Destroy the cluster } var ( diff --git a/pkg/automation/baremetal.go b/pkg/automation/baremetal.go index 7939369..10d5524 100644 --- a/pkg/automation/baremetal.go +++ b/pkg/automation/baremetal.go @@ -35,6 +35,14 @@ type scriptRunInstance struct { args []string } +type terraformOperation string + +const ( + terraformApply terraformOperation = "apply" + terraformDestroy terraformOperation = "destroy" + terraformInit terraformOperation = "init" +) + func newBaremetal(params AutomatedDeploymentParams) (AutomatedDeploymentInterface, error) { // Examine site's site-config and determine if automation is even possible for this site siteConfigSourcePath := fmt.Sprintf("%s/%s/site/00_install-config/site-config.yaml", params.SiteBuildPath, params.SiteName) @@ -197,7 +205,7 @@ func (bad baremetalAutomatedDeployment) DeployMasters() error { } // Finally run terraform commands to begin cluster deployment - err = bad.runTerraform(automationRepoPath, "cluster") + err = bad.runTerraform(automationRepoPath, "cluster", terraformApply) if err != nil { return err @@ -352,7 +360,7 @@ func (bad baremetalAutomatedDeployment) DeployWorkers() error { } // Finally run terraform commands to begin workers deployment - err = bad.runTerraform(automationRepoPath, "workers") + err = bad.runTerraform(automationRepoPath, "workers", terraformApply) if err != nil { return err @@ -363,13 +371,95 @@ func (bad baremetalAutomatedDeployment) DeployWorkers() error { return nil } -func (bad baremetalAutomatedDeployment) runTerraform(automationRepoPath string, targetType string) error { +func (bad baremetalAutomatedDeployment) DestroyCluster() error { + sitePath := fmt.Sprintf("%s/%s", bad.siteBuildPath, bad.siteName) + automationRepoPath := fmt.Sprintf("%s/baremetal_automation", sitePath) + + _, err := os.Stat(automationRepoPath) + + if err != nil { + return fmt.Errorf("baremetalAutomatedDeployment: DestroyCluster: unable to access local automation repo at %s: %s", automationRepoPath, err) + } + + // Destroy workers via terraform + err = bad.runTerraform(automationRepoPath, "workers", terraformDestroy) + + if err != nil { + return err + } + + // Destroy masters via terraform + // TODO: Ignoring errors here until we fix the bogus bootstrap VM destruction error + // that falsely reports a problem when there isn't one (the error says that the + // VM cannot be found, but this is expected because the VM was just destroyed!) + bad.runTerraform(automationRepoPath, "cluster", terraformDestroy) + + // Remove bastion (provisioning host) containers + scripts := []scriptRunInstance{} + + commonArgs := []string{"remove"} + + scripts = append(scripts, scriptRunInstance{ + description: "dnsmasq provisioning container removal", + scriptFile: "gen_config_prov.sh", + args: commonArgs, + }) + + scripts = append(scripts, scriptRunInstance{ + description: "dnsmasq baremetal container removal", + scriptFile: "gen_config_bm.sh", + args: commonArgs, + }) + + scripts = append(scripts, scriptRunInstance{ + description: "haproxy container removal", + scriptFile: "gen_haproxy.sh", + args: commonArgs, + }) + + scripts = append(scripts, scriptRunInstance{ + description: "coredns container removal", + scriptFile: "gen_coredns.sh", + args: commonArgs, + }) + + scripts = append(scripts, scriptRunInstance{ + description: "matchbox container removal", + scriptFile: "gen_matchbox.sh", + args: commonArgs, + }) + + err = bad.runScripts(automationRepoPath, scripts) + + if err != nil { + return err + } + + // Clear config directories + dirs := []string{ + "build", + "coredns", + "dnsmasq", + "haproxy", + "ocp", + } + + for _, dir := range dirs { + os.RemoveAll(fmt.Sprintf("%s/%s", automationRepoPath, dir)) + } + + log.Printf("baremetalAutomatedDeployment: DestroyCluster: cluster teardown completed\n") + + return nil +} + +func (bad baremetalAutomatedDeployment) runTerraform(automationRepoPath string, targetType string, operation terraformOperation) error { terraformPath := fmt.Sprintf("%s/terraform/%s", automationRepoPath, targetType) log.Printf("baremetalAutomatedDeployment: runTerraform: initializing terraform...\n") // Init - cmd := exec.Command("terraform", "init") + cmd := exec.Command("terraform", string(terraformInit)) cmd.Dir = terraformPath cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -381,10 +471,10 @@ func (bad baremetalAutomatedDeployment) runTerraform(automationRepoPath string, } log.Printf("baremetalAutomatedDeployment: runTerraform: terraform successfully initialized\n") - log.Printf("baremetalAutomatedDeployment: runTerraform: applying terraform...\n") + log.Printf("baremetalAutomatedDeployment: runTerraform: running terraform %s...\n", operation) // Apply - cmd = exec.Command("terraform", "apply", "--auto-approve") + cmd = exec.Command("terraform", string(operation), "--auto-approve") cmd.Dir = terraformPath cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -392,7 +482,7 @@ func (bad baremetalAutomatedDeployment) runTerraform(automationRepoPath string, err = cmd.Run() if err != nil { - return fmt.Errorf("baremetalAutomatedDeployment: runTerraform: error running baremetal automation %s terraform apply: %s", targetType, err) + return fmt.Errorf("baremetalAutomatedDeployment: runTerraform: error running baremetal automation %s terraform %s: %s", targetType, operation, err) } log.Printf("baremetalAutomatedDeployment: runTerraform: terraform successfully applied\n") diff --git a/pkg/site/site.go b/pkg/site/site.go index 6e788a6..4a44ad9 100644 --- a/pkg/site/site.go +++ b/pkg/site/site.go @@ -488,18 +488,55 @@ func (s Site) AutomateWorkersDeployment() { } } +func (s Site) AutomateClusterDestroy() { + // Get an automated deployment object + automatedDeployment, err := s.getAutomatedDeployment() + + if err != nil { + log.Fatal(fmt.Sprintf("Site: AutomateClusterDestroy: Error attempting to acquire automated deploy object: %s", err)) + os.Exit(1) + } + + // Run the automated cluster teardown + err = automatedDeployment.DestroyCluster() + + if err != nil { + log.Fatal(fmt.Sprintf("Site: AutomateClusterDestroy: Error attempting to run automated cluster destroy: %s", err)) + os.Exit(1) + } +} + func (s Site) automateDeployment(deploymentType string) error { + // Get an automated deployment object + automatedDeployment, err := s.getAutomatedDeployment() + + if err != nil { + return err + } + + // Act based on the requested deployment type + switch deploymentType { + case "masters": + return automatedDeployment.DeployMasters() + case "workers": + return automatedDeployment.DeployWorkers() + default: + return fmt.Errorf("Site: automateDeployment: unknown deployment type: %s", deploymentType) + } +} + +// Returns an AutomatedDeploymentInterface for use with automation operations +func (s Site) getAutomatedDeployment() (automation.AutomatedDeploymentInterface, error) { // Get profile name profileName, _, _ := s.GetProfileFromSite() // Get the profile type - // NOTE: This also checks whether the site repo exists locally, so there is no + // NOTE: This call also checks whether the site repo exists locally, so there is no // need to check that here profileType, err := s.getProfileType(profileName) if err != nil { - log.Fatal(fmt.Sprintf("Site: automateDeployment: Error acquiring site profile type: %s", err)) - os.Exit(1) + return nil, fmt.Errorf("Site: getAutomatedDeployment: Error acquiring site profile type: %s", err) } // Create an automated deployment instance @@ -513,26 +550,16 @@ func (s Site) automateDeployment(deploymentType string) error { automatedDeployment, err := automation.New(automatedDeploymentParams) if err != nil { - log.Fatal(fmt.Sprintf("Site: automateDeployment: Error creating automated deployment instance: %s", err)) - os.Exit(1) + return nil, fmt.Errorf("Site: getAutomatedDeployment: Error creating automated deployment instance: %s", err) } // If nil is returned for automatedDeployment, then this particular site does // not contain the necessary config required to automate its deployment if automatedDeployment == nil { - return fmt.Errorf("Site: automateDeployment: automated deployment not supported for site '%s'", s.siteName) - } - - switch deploymentType { - case "masters": - return automatedDeployment.DeployMasters() - case "workers": - return automatedDeployment.DeployWorkers() - default: - return fmt.Errorf("Site: automateDeployment: unknown deployment type: %s", deploymentType) + return nil, fmt.Errorf("Site: getAutomatedDeployment: automated deployment not supported for site '%s'", s.siteName) } - return nil + return automatedDeployment, nil } // Determines site profile type based on blueprint profile contents @@ -641,28 +668,21 @@ func (s Site) prepareHostForAutomation(profileName string) error { return errors.New("Site: prepareHostForAutomation: build path and/or site name missing") } - // Determine profile type - profileType, err := s.getProfileType(profileName) - - if err != nil { - return err - } + // Clear any existing automation folders + automationManifests := fmt.Sprintf("%s/%s/automation", s.buildPath, s.siteName) + automationDestination := fmt.Sprintf("%s/%s/baremetal_automation", s.buildPath, s.siteName) - // Attempt to create an automated deployment instance - automatedDeploymentParams := automation.AutomatedDeploymentParams{ - ProfileType: profileType, - SiteBuildPath: s.buildPath, - SiteName: s.siteName, - SiteRepo: s.siteRepo, - } + os.RemoveAll(automationManifests) + os.RemoveAll(automationDestination) - automatedDeployment, err := automation.New(automatedDeploymentParams) + // Get an automated deployment object + automatedDeployment, err := s.getAutomatedDeployment() if err != nil { // If automation isn't supported for this profile type, it's not a fatal error in // this context, since this function is just trying to prepare the host for potential // automation (and is not called in the context of an explicit automation request) - if strings.Contains(err.Error(), "automation not supported") { + if strings.Contains(err.Error(), "automation not supported") || strings.Contains(err.Error(), "automated deployment not supported") { return nil }