Automated cluster teardown command 74/1774/1
authorAndrew Bays <abays@redhat.com>
Fri, 11 Oct 2019 19:32:57 +0000 (15:32 -0400)
committerAndrew Bays <abays@redhat.com>
Fri, 11 Oct 2019 19:32:57 +0000 (15:32 -0400)
Change-Id: Id4d4df957b08a1b38b50fbe8bd948904f82a936f

cmd/deploy_masters.go
cmd/deploy_workers.go
cmd/destroy_cluster.go [new file with mode: 0644]
pkg/automation/automation.go
pkg/automation/baremetal.go
pkg/site/site.go

index 6683ae3..7194978 100644 (file)
@@ -23,7 +23,7 @@ import (
        "github.com/spf13/cobra"
 )
 
-// deployMastersCmd represents the automate_masters_deployment command
+// deployMastersCmd represents the deploy_masters command
 var deployMastersCmd = &cobra.Command{
        Use:              "deploy_masters siteName [--build_path=<local_build_path>]",
        Short:            "Command to automate the deployment of the master nodes of a previously-prepared site",
index 20b9881..c547303 100644 (file)
@@ -23,7 +23,7 @@ import (
        "github.com/spf13/cobra"
 )
 
-// deployWorkersCmd represents the automate_workers_deployment command
+// deployWorkersCmd represents the deploy_workers command
 var deployWorkersCmd = &cobra.Command{
        Use:              "deploy_workers siteName [--build_path=<local_build_path>]",
        Short:            "Command to automate the deployment of the worker nodes of a previously-prepared site",
diff --git a/cmd/destroy_cluster.go b/cmd/destroy_cluster.go
new file mode 100644 (file)
index 0000000..747372f
--- /dev/null
@@ -0,0 +1,60 @@
+// Copyright © 2019 Red Hat <abays@redhat.com>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+       "fmt"
+       "log"
+       "os"
+
+       "gerrit.akraino.org/kni/installer/pkg/site"
+       "github.com/spf13/cobra"
+)
+
+// destroyClusterCmd represents the destroy_cluster command
+var destroyClusterCmd = &cobra.Command{
+       Use:              "destroy_cluster siteName [--build_path=<local_build_path>]",
+       Short:            "Command to automate the teardown of master and workers nodes of an automated-deployment cluster",
+       Long:             ``,
+       TraverseChildren: true,
+       Run: func(cmd *cobra.Command, args []string) {
+               // retrieve config values and start fetching
+               var siteName string
+               if len(args) == 0 {
+                       log.Fatal("Please specify site name as first argument")
+                       os.Exit(1)
+               } else {
+                       siteName = args[0]
+               }
+
+               buildPath, _ := cmd.Flags().GetString("build_path")
+               if len(buildPath) == 0 {
+                       // will generate a temporary directory
+                       buildPath = fmt.Sprintf("%s/.kni", os.Getenv("HOME"))
+               }
+
+               // This command is used after fetch_requirements and prepare_manifests,
+               // so the site directory should be available on disk already (if not,
+               // s.AutomateMastersDeployment will error-out appropriately)
+               s := site.NewWithName(siteName, buildPath)
+               s.AutomateClusterDestroy()
+       },
+}
+
+func init() {
+       rootCmd.AddCommand(destroyClusterCmd)
+
+       destroyClusterCmd.Flags().StringP("build_path", "", "", "Directory to use as build path. If that doesn't exist, the installer will generate a default directory")
+}
index 3fdf891..e5c4304 100644 (file)
@@ -19,6 +19,7 @@ type AutomatedDeploymentInterface interface {
        PrepareBastion() error // Prepare host for automation
        DeployMasters() error  // Deploy cluster masters
        DeployWorkers() error  // Deploy cluster workers
+       DestroyCluster() error // Destroy the cluster
 }
 
 var (
index 7939369..10d5524 100644 (file)
@@ -35,6 +35,14 @@ type scriptRunInstance struct {
        args        []string
 }
 
+type terraformOperation string
+
+const (
+       terraformApply   terraformOperation = "apply"
+       terraformDestroy terraformOperation = "destroy"
+       terraformInit    terraformOperation = "init"
+)
+
 func newBaremetal(params AutomatedDeploymentParams) (AutomatedDeploymentInterface, error) {
        // Examine site's site-config and determine if automation is even possible for this site
        siteConfigSourcePath := fmt.Sprintf("%s/%s/site/00_install-config/site-config.yaml", params.SiteBuildPath, params.SiteName)
@@ -197,7 +205,7 @@ func (bad baremetalAutomatedDeployment) DeployMasters() error {
        }
 
        // Finally run terraform commands to begin cluster deployment
-       err = bad.runTerraform(automationRepoPath, "cluster")
+       err = bad.runTerraform(automationRepoPath, "cluster", terraformApply)
 
        if err != nil {
                return err
@@ -352,7 +360,7 @@ func (bad baremetalAutomatedDeployment) DeployWorkers() error {
        }
 
        // Finally run terraform commands to begin workers deployment
-       err = bad.runTerraform(automationRepoPath, "workers")
+       err = bad.runTerraform(automationRepoPath, "workers", terraformApply)
 
        if err != nil {
                return err
@@ -363,13 +371,95 @@ func (bad baremetalAutomatedDeployment) DeployWorkers() error {
        return nil
 }
 
-func (bad baremetalAutomatedDeployment) runTerraform(automationRepoPath string, targetType string) error {
+func (bad baremetalAutomatedDeployment) DestroyCluster() error {
+       sitePath := fmt.Sprintf("%s/%s", bad.siteBuildPath, bad.siteName)
+       automationRepoPath := fmt.Sprintf("%s/baremetal_automation", sitePath)
+
+       _, err := os.Stat(automationRepoPath)
+
+       if err != nil {
+               return fmt.Errorf("baremetalAutomatedDeployment: DestroyCluster: unable to access local automation repo at %s: %s", automationRepoPath, err)
+       }
+
+       // Destroy workers via terraform
+       err = bad.runTerraform(automationRepoPath, "workers", terraformDestroy)
+
+       if err != nil {
+               return err
+       }
+
+       // Destroy masters via terraform
+       // TODO: Ignoring errors here until we fix the bogus bootstrap VM destruction error
+       //       that falsely reports a problem when there isn't one (the error says that the
+       //       VM cannot be found, but this is expected because the VM was just destroyed!)
+       bad.runTerraform(automationRepoPath, "cluster", terraformDestroy)
+
+       // Remove bastion (provisioning host) containers
+       scripts := []scriptRunInstance{}
+
+       commonArgs := []string{"remove"}
+
+       scripts = append(scripts, scriptRunInstance{
+               description: "dnsmasq provisioning container removal",
+               scriptFile:  "gen_config_prov.sh",
+               args:        commonArgs,
+       })
+
+       scripts = append(scripts, scriptRunInstance{
+               description: "dnsmasq baremetal container removal",
+               scriptFile:  "gen_config_bm.sh",
+               args:        commonArgs,
+       })
+
+       scripts = append(scripts, scriptRunInstance{
+               description: "haproxy container removal",
+               scriptFile:  "gen_haproxy.sh",
+               args:        commonArgs,
+       })
+
+       scripts = append(scripts, scriptRunInstance{
+               description: "coredns container removal",
+               scriptFile:  "gen_coredns.sh",
+               args:        commonArgs,
+       })
+
+       scripts = append(scripts, scriptRunInstance{
+               description: "matchbox container removal",
+               scriptFile:  "gen_matchbox.sh",
+               args:        commonArgs,
+       })
+
+       err = bad.runScripts(automationRepoPath, scripts)
+
+       if err != nil {
+               return err
+       }
+
+       // Clear config directories
+       dirs := []string{
+               "build",
+               "coredns",
+               "dnsmasq",
+               "haproxy",
+               "ocp",
+       }
+
+       for _, dir := range dirs {
+               os.RemoveAll(fmt.Sprintf("%s/%s", automationRepoPath, dir))
+       }
+
+       log.Printf("baremetalAutomatedDeployment: DestroyCluster: cluster teardown completed\n")
+
+       return nil
+}
+
+func (bad baremetalAutomatedDeployment) runTerraform(automationRepoPath string, targetType string, operation terraformOperation) error {
        terraformPath := fmt.Sprintf("%s/terraform/%s", automationRepoPath, targetType)
 
        log.Printf("baremetalAutomatedDeployment: runTerraform: initializing terraform...\n")
 
        // Init
-       cmd := exec.Command("terraform", "init")
+       cmd := exec.Command("terraform", string(terraformInit))
        cmd.Dir = terraformPath
        cmd.Stdout = os.Stdout
        cmd.Stderr = os.Stderr
@@ -381,10 +471,10 @@ func (bad baremetalAutomatedDeployment) runTerraform(automationRepoPath string,
        }
 
        log.Printf("baremetalAutomatedDeployment: runTerraform: terraform successfully initialized\n")
-       log.Printf("baremetalAutomatedDeployment: runTerraform: applying terraform...\n")
+       log.Printf("baremetalAutomatedDeployment: runTerraform: running terraform %s...\n", operation)
 
        // Apply
-       cmd = exec.Command("terraform", "apply", "--auto-approve")
+       cmd = exec.Command("terraform", string(operation), "--auto-approve")
        cmd.Dir = terraformPath
        cmd.Stdout = os.Stdout
        cmd.Stderr = os.Stderr
@@ -392,7 +482,7 @@ func (bad baremetalAutomatedDeployment) runTerraform(automationRepoPath string,
        err = cmd.Run()
 
        if err != nil {
-               return fmt.Errorf("baremetalAutomatedDeployment: runTerraform: error running baremetal automation %s terraform apply: %s", targetType, err)
+               return fmt.Errorf("baremetalAutomatedDeployment: runTerraform: error running baremetal automation %s terraform %s: %s", targetType, operation, err)
        }
 
        log.Printf("baremetalAutomatedDeployment: runTerraform: terraform successfully applied\n")
index 6e788a6..4a44ad9 100644 (file)
@@ -488,18 +488,55 @@ func (s Site) AutomateWorkersDeployment() {
        }
 }
 
+func (s Site) AutomateClusterDestroy() {
+       // Get an automated deployment object
+       automatedDeployment, err := s.getAutomatedDeployment()
+
+       if err != nil {
+               log.Fatal(fmt.Sprintf("Site: AutomateClusterDestroy: Error attempting to acquire automated deploy object: %s", err))
+               os.Exit(1)
+       }
+
+       // Run the automated cluster teardown
+       err = automatedDeployment.DestroyCluster()
+
+       if err != nil {
+               log.Fatal(fmt.Sprintf("Site: AutomateClusterDestroy: Error attempting to run automated cluster destroy: %s", err))
+               os.Exit(1)
+       }
+}
+
 func (s Site) automateDeployment(deploymentType string) error {
+       // Get an automated deployment object
+       automatedDeployment, err := s.getAutomatedDeployment()
+
+       if err != nil {
+               return err
+       }
+
+       // Act based on the requested deployment type
+       switch deploymentType {
+       case "masters":
+               return automatedDeployment.DeployMasters()
+       case "workers":
+               return automatedDeployment.DeployWorkers()
+       default:
+               return fmt.Errorf("Site: automateDeployment: unknown deployment type: %s", deploymentType)
+       }
+}
+
+// Returns an AutomatedDeploymentInterface for use with automation operations
+func (s Site) getAutomatedDeployment() (automation.AutomatedDeploymentInterface, error) {
        // Get profile name
        profileName, _, _ := s.GetProfileFromSite()
 
        // Get the profile type
-       // NOTE: This also checks whether the site repo exists locally, so there is no
+       // NOTE: This call also checks whether the site repo exists locally, so there is no
        //       need to check that here
        profileType, err := s.getProfileType(profileName)
 
        if err != nil {
-               log.Fatal(fmt.Sprintf("Site: automateDeployment: Error acquiring site profile type: %s", err))
-               os.Exit(1)
+               return nil, fmt.Errorf("Site: getAutomatedDeployment: Error acquiring site profile type: %s", err)
        }
 
        // Create an automated deployment instance
@@ -513,26 +550,16 @@ func (s Site) automateDeployment(deploymentType string) error {
        automatedDeployment, err := automation.New(automatedDeploymentParams)
 
        if err != nil {
-               log.Fatal(fmt.Sprintf("Site: automateDeployment: Error creating automated deployment instance: %s", err))
-               os.Exit(1)
+               return nil, fmt.Errorf("Site: getAutomatedDeployment: Error creating automated deployment instance: %s", err)
        }
 
        // If nil is returned for automatedDeployment, then this particular site does
        // not contain the necessary config required to automate its deployment
        if automatedDeployment == nil {
-               return fmt.Errorf("Site: automateDeployment: automated deployment not supported for site '%s'", s.siteName)
-       }
-
-       switch deploymentType {
-       case "masters":
-               return automatedDeployment.DeployMasters()
-       case "workers":
-               return automatedDeployment.DeployWorkers()
-       default:
-               return fmt.Errorf("Site: automateDeployment: unknown deployment type: %s", deploymentType)
+               return nil, fmt.Errorf("Site: getAutomatedDeployment: automated deployment not supported for site '%s'", s.siteName)
        }
 
-       return nil
+       return automatedDeployment, nil
 }
 
 // Determines site profile type based on blueprint profile contents
@@ -641,28 +668,21 @@ func (s Site) prepareHostForAutomation(profileName string) error {
                return errors.New("Site: prepareHostForAutomation: build path and/or site name missing")
        }
 
-       // Determine profile type
-       profileType, err := s.getProfileType(profileName)
-
-       if err != nil {
-               return err
-       }
+       // Clear any existing automation folders
+       automationManifests := fmt.Sprintf("%s/%s/automation", s.buildPath, s.siteName)
+       automationDestination := fmt.Sprintf("%s/%s/baremetal_automation", s.buildPath, s.siteName)
 
-       // Attempt to create an automated deployment instance
-       automatedDeploymentParams := automation.AutomatedDeploymentParams{
-               ProfileType:   profileType,
-               SiteBuildPath: s.buildPath,
-               SiteName:      s.siteName,
-               SiteRepo:      s.siteRepo,
-       }
+       os.RemoveAll(automationManifests)
+       os.RemoveAll(automationDestination)
 
-       automatedDeployment, err := automation.New(automatedDeploymentParams)
+       // Get an automated deployment object
+       automatedDeployment, err := s.getAutomatedDeployment()
 
        if err != nil {
                // If automation isn't supported for this profile type, it's not a fatal error in
                // this context, since this function is just trying to prepare the host for potential
                // automation (and is not called in the context of an explicit automation request)
-               if strings.Contains(err.Error(), "automation not supported") {
+               if strings.Contains(err.Error(), "automation not supported") || strings.Contains(err.Error(), "automated deployment not supported") {
                        return nil
                }